diff options
| author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:35:22 +0100 |
|---|---|---|
| committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 20:35:22 +0100 |
| commit | e0401df1214397626e65e58166988fe62715d372 (patch) | |
| tree | 087d8ca4a1611aa50a8ac98e66f7d1657ff1f90f /lib | |
| parent | 2b7b60728426c10ef1188a1073d3630805773a35 (diff) | |
| parent | 11c67e7c2f992299ff5918ce67995b73d1e0be6d (diff) | |
| download | gitlab-ce-e0401df1214397626e65e58166988fe62715d372.tar.gz | |
Merge commit '11c67e7c2f992299ff5918ce67995b73d1e0be6d' into object-storage-ee-to-ce-backport
Diffstat (limited to 'lib')
183 files changed, 5243 insertions, 1767 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index ee4e1688e12..79e55a2f4f7 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -8,7 +8,6 @@ module API logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ - GrapeLogging::Loggers::Response.new, GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new ] diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c4c0fdda665..e79f988f549 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -75,7 +75,7 @@ module API raise RevokedError when AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + User.find(access_token.resource_owner_id) end end @@ -84,11 +84,13 @@ module API return nil unless token_string.present? - find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) - end + user = + find_user_by_authentication_token(token_string) || + find_user_by_personal_access_token(token_string, scopes) + + raise UnauthorizedError unless user - def current_user - @current_user + user end private @@ -107,7 +109,16 @@ module API end def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + return @access_token if defined?(@access_token) + + token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + return @access_token = nil unless token + + @access_token = Doorkeeper::AccessToken.by_token(token) + raise UnauthorizedError unless @access_token + + @access_token.revoke_previous_refresh_token! + @access_token end def doorkeeper_request @@ -169,6 +180,7 @@ module API TokenNotFoundError = Class.new(StandardError) ExpiredError = Class.new(StandardError) RevokedError = Class.new(StandardError) + UnauthorizedError = Class.new(StandardError) class InsufficientScopeError < StandardError attr_reader :scopes diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 642c1140fcc..61a2d688282 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -13,7 +13,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository branches' do - success Entities::RepoBranch + success Entities::Branch end params do use :pagination @@ -21,12 +21,15 @@ module API get ':id/repository/branches' do branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) - present paginate(branches), with: Entities::RepoBranch, project: user_project + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 + Gitlab::GitalyClient.allow_n_plus_1_calls do + present paginate(branches), with: Entities::Branch, project: user_project + end end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do desc 'Get a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -38,7 +41,7 @@ module API branch = user_project.repository.find_branch(params[:branch]) not_found!('Branch') unless branch - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end end @@ -47,7 +50,7 @@ module API # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. desc 'Protect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -77,7 +80,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -85,7 +88,7 @@ module API # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -98,11 +101,11 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::Branch, project: user_project end desc 'Create branch' do - success Entities::RepoBranch + success Entities::Branch end params do requires :branch, type: String, desc: 'The name of the branch' @@ -116,7 +119,7 @@ module API if result[:status] == :success present result[:branch], - with: Entities::RepoBranch, + with: Entities::Branch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4b8d248f5f7..4af37a2ad1d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -13,7 +13,7 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do - success Entities::RepoCommit + success Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -46,11 +46,11 @@ module API paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present paginate(paginated_commits), with: Entities::RepoCommit + present paginate(paginated_commits), with: Entities::Commit end desc 'Commit multiple file changes as one commit' do - success Entities::RepoCommitDetail + success Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -72,14 +72,14 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - present commit_detail, with: Entities::RepoCommitDetail + present commit_detail, with: Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success Entities::RepoCommitDetail + success Entities::CommitDetail failure [[404, 'Commit Not Found']] end params do @@ -90,7 +90,7 @@ module API not_found! 'Commit' unless commit - present commit, with: Entities::RepoCommitDetail + present commit, with: Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -104,7 +104,7 @@ module API not_found! 'Commit' unless commit - present commit.raw_diffs.to_a, with: Entities::RepoDiff + present commit.raw_diffs.to_a, with: Entities::Diff end desc "Get a commit's comments" do @@ -126,7 +126,7 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success Entities::RepoCommit + success Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' @@ -151,7 +151,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit else render_api_error!(result[:message], 400) end diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb new file mode 100644 index 00000000000..5000aa0d9ac --- /dev/null +++ b/lib/api/custom_attributes_endpoints.rb @@ -0,0 +1,77 @@ +module API + module CustomAttributesEndpoints + extend ActiveSupport::Concern + + included do + attributable_class = name.demodulize.singularize + attributable_key = attributable_class.underscore + attributable_name = attributable_class.humanize(capitalize: false) + attributable_finder = "find_#{attributable_key}" + + helpers do + params :custom_attributes_key do + requires :key, type: String, desc: 'The key of the custom attribute' + end + end + + desc "Get all custom attributes on a #{attributable_name}" do + success Entities::CustomAttribute + end + get ':id/custom_attributes' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + present resource.custom_attributes, with: Entities::CustomAttribute + end + + desc "Get a custom attribute on a #{attributable_name}" do + success Entities::CustomAttribute + end + params do + use :custom_attributes_key + end + get ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + custom_attribute = resource.custom_attributes.find_by!(key: params[:key]) + + present custom_attribute, with: Entities::CustomAttribute + end + + desc "Set a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + requires :value, type: String, desc: 'The value of the custom attribute' + end + put ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + custom_attribute = resource.custom_attributes + .find_or_initialize_by(key: params[:key]) + + custom_attribute.update(value: params[:value]) + + if custom_attribute.valid? + present custom_attribute, with: Entities::CustomAttribute + else + render_validation_error!(custom_attribute) + end + end + + desc "Delete a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + end + delete ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + resource.custom_attributes.find_by!(key: params[:key]).destroy + + status 204 + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 52c49e5caa9..5f0bad14839 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -89,6 +89,9 @@ module API expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :avatar_url do |project, options| + project.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :created_at, :last_activity_at end @@ -146,9 +149,7 @@ module API expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs @@ -193,8 +194,8 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) + expose :avatar_url do |group, options| + group.avatar_url(only_path: false) end expose :web_url expose :request_access_enabled @@ -219,7 +220,7 @@ module API expose :shared_projects, using: Entities::Project end - class RepoCommit < Grape::Entity + class Commit < Grape::Entity expose :id, :short_id, :title, :created_at expose :parent_ids expose :safe_message, as: :message @@ -227,24 +228,28 @@ module API expose :committer_name, :committer_email, :committed_date end - class RepoCommitStats < Grape::Entity + class CommitStats < Grape::Entity expose :additions, :deletions, :total end - class RepoCommitDetail < RepoCommit - expose :stats, using: Entities::RepoCommitStats + class CommitDetail < Commit + expose :stats, using: Entities::CommitStats expose :status + expose :last_pipeline, using: 'API::Entities::PipelineBasic' end - class RepoBranch < Grape::Entity + class Branch < Grape::Entity expose :name - expose :commit, using: Entities::RepoCommit do |repo_branch, options| + expose :commit, using: Entities::Commit do |repo_branch, options| options[:project].repository.commit(repo_branch.dereferenced_target) end expose :merged do |repo_branch, options| - options[:project].repository.merged_to_root_ref?(repo_branch.name) + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 + Gitlab::GitalyClient.allow_n_plus_1_calls do + options[:project].repository.merged_to_root_ref?(repo_branch.name) + end end expose :protected do |repo_branch, options| @@ -260,7 +265,7 @@ module API end end - class RepoTreeObject < Grape::Entity + class TreeObject < Grape::Entity expose :id, :name, :type, :path expose :mode do |obj, options| @@ -300,7 +305,7 @@ module API expose :state, :created_at, :updated_at end - class RepoDiff < Grape::Entity + class Diff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode expose :new_file?, as: :new_file expose :renamed_file?, as: :renamed_file @@ -332,6 +337,7 @@ module API end class IssueBasic < ProjectEntity + expose :closed_at expose :labels do |issue, options| # Avoids an N+1 query since labels are preloaded issue.labels.map(&:title).sort @@ -362,6 +368,7 @@ module API end expose :due_date expose :confidential + expose :discussion_locked expose :web_url do |issue, options| Gitlab::UrlBuilder.build(issue) @@ -458,6 +465,7 @@ module API expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :user_notes_count + expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -477,7 +485,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -488,9 +496,9 @@ module API end class MergeRequestDiffFull < MergeRequestDiff - expose :commits, using: Entities::RepoCommit + expose :commits, using: Entities::Commit - expose :diffs, using: Entities::RepoDiff do |compare, _| + expose :diffs, using: Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end @@ -586,8 +594,7 @@ module API expose :target_type expose :target do |todo, options| - target = todo.target_type == 'Commit' ? 'RepoCommit' : todo.target_type - Entities.const_get(target).represent(todo.target, options) + Entities.const_get(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| @@ -723,15 +730,15 @@ module API end class Compare < Grape::Entity - expose :commit, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil).last + expose :commit, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil).last end - expose :commits, using: Entities::RepoCommit do |compare, options| - Commit.decorate(compare.commits, nil) + expose :commits, using: Entities::Commit do |compare, options| + ::Commit.decorate(compare.commits, nil) end - expose :diffs, using: Entities::RepoDiff do |compare, options| + expose :diffs, using: Entities::Diff do |compare, options| compare.diffs(limits: false).to_a end @@ -767,10 +774,10 @@ module API expose :description end - class RepoTag < Grape::Entity + class Tag < Grape::Entity expose :name, :message - expose :commit, using: Entities::RepoCommit do |repo_tag, options| + expose :commit, using: Entities::Commit do |repo_tag, options| options[:project].repository.commit(repo_tag.dereferenced_target) end @@ -821,7 +828,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: User expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } - expose :commit, with: RepoCommit + expose :commit, with: Commit expose :runner, with: Runner expose :pipeline, with: PipelineBasic end @@ -874,7 +881,7 @@ module API expose :deployable, using: Entities::Job end - class RepoLicense < Grape::Entity + class License < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular expose :url, as: :html_url @@ -1016,6 +1023,7 @@ module API expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency + expose :features end end @@ -1030,5 +1038,10 @@ module API expose :failing_on_hosts expose :total_failures end + + class CustomAttribute < Grape::Entity + expose :key + expose :value + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6a0be74623e..67ef329f01f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -3,6 +3,8 @@ module API include Gitlab::Utils include Helpers::Pagination + UnauthorizedError = Class.new(StandardError) + SUDO_HEADER = "HTTP_SUDO".freeze SUDO_PARAM = :sudo @@ -139,7 +141,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(initial_current_user, :access_api) + unauthorized! unless current_user end def authenticate_non_get! @@ -285,7 +287,7 @@ module API if sentry_enabled? && report_exception?(exception) define_params_for_grape_middleware sentry_context - Raven.capture_exception(exception) + Raven.capture_exception(exception, extra: params) end # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 @@ -397,19 +399,27 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) - Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= find_user_from_warden - - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil - end - @initial_current_user + begin + @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user } + rescue APIGuard::UnauthorizedError, UnauthorizedError + unauthorized! end end + def find_current_user + user = + find_user_by_private_token(scopes: scopes_registered_for_endpoint) || + doorkeeper_guard(scopes: scopes_registered_for_endpoint) || + find_user_from_warden + + return nil unless user + + raise UnauthorizedError unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + + user + end + def sudo! return unless sudo_identifier return unless initial_current_user @@ -454,10 +464,12 @@ module API header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) end - # The Grape Error Middleware only has access to env but no params. We workaround this by - # defining a method that returns the right value. + # The Grape Error Middleware only has access to `env` but not `params` nor + # `request`. We workaround this by defining methods that returns the right + # values. def define_params_for_grape_middleware - self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:params) { request.params.symbolize_keys } end # We could get a Grape or a standard Ruby exception. We should only report anything that diff --git a/lib/api/internal.rb b/lib/api/internal.rb index c0fef56378f..6e78ac2c903 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -31,6 +31,12 @@ module API protocol = params[:protocol] actor.update_last_used_at if actor.is_a?(Key) + user = + if actor.is_a?(Key) + actor.user + else + actor + end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass @@ -47,6 +53,7 @@ module API { status: true, gl_repository: gl_repository, + gl_username: user&.username, repository_path: repository_path, gitaly: gitaly_payload(params[:action]) } @@ -136,7 +143,7 @@ module API codes = nil - ::Users::UpdateService.new(user).execute! do |user| + ::Users::UpdateService.new(current_user, user: user).execute! do |user| codes = user.generate_otp_backup_codes! end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1729df2aad0..0df41dcc903 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -48,6 +48,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" end params :issue_params do @@ -193,7 +194,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index ae43a4a3237..d202eaa4c49 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -6,7 +6,7 @@ module API requires :content, type: String, desc: 'Content of .gitlab-ci.yml' end post '/lint' do - error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) status 200 diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 56d72d511da..be843ec8251 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -2,7 +2,7 @@ module API class MergeRequests < Grape::API include PaginationParams - before { authenticate! } + before { authenticate_non_get! } helpers ::Gitlab::IssuableMetadata @@ -55,6 +55,7 @@ module API desc: 'Return merge requests for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do + authenticate! unless params[:scope] == 'all' merge_requests = find_merge_requests options = { with: Entities::MergeRequestBasic, @@ -182,13 +183,13 @@ module API end desc 'Get the commits of a merge request' do - success Entities::RepoCommit + success Entities::Commit end get ':id/merge_requests/:merge_request_iid/commits' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = ::Kaminari.paginate_array(merge_request.commits) - present paginate(commits), with: Entities::RepoCommit + present paginate(commits), with: Entities::Commit end desc 'Show the merge request changes' do @@ -213,12 +214,14 @@ module API :remove_source_branch, :state_event, :target_branch, - :title + :title, + :discussion_locked ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], desc: 'Status of the merge request' + optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' use :optional_params at_least_one_of(*at_least_one_of_ce) diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d6e7203adaf..0b9ab4eeb05 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -78,6 +78,8 @@ module API } if can?(current_user, noteable_read_ability_name(noteable), noteable) + authorize! :create_note, noteable + if params[:created_at] && (current_user.admin? || user_project.owner == current_user) opts[:created_at] = params[:created_at] end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bcc0833aa5c..0266bf2f717 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -35,7 +35,7 @@ module API new_notification_email = params.delete(:notification_email) if new_notification_email - ::Users::UpdateService.new(current_user, notification_email: new_notification_email).execute + ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute end notification_setting.update(declared_params(include_missing: false)) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dc19788462..aab7a6c3f93 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -70,8 +70,11 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(options = {}) - projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + def load_projects + ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + end + + def present_projects(projects, options = {}) projects = reorder_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] @@ -111,7 +114,7 @@ module API params[:user] = user - present_projects + present_projects load_projects end end @@ -124,7 +127,7 @@ module API use :statistics_params end get do - present_projects + present_projects load_projects end desc 'Create new project' do @@ -229,6 +232,18 @@ module API end end + desc 'List forks of this project' do + success Entities::Project + end + params do + use :collection_params + end + get ':id/forks' do + forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute + + present_projects forks + end + desc 'Update an existing project' do success Entities::Project end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 2255fb1b70d..ceee3226732 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -35,7 +35,7 @@ module API end desc 'Get a project repository tree' do - success Entities::RepoTreeObject + success Entities::TreeObject end params do optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -52,7 +52,7 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) entries = ::Kaminari.paginate_array(tree.sorted_entries) - present paginate(entries), with: Entities::RepoTreeObject + present paginate(entries), with: Entities::TreeObject end desc 'Get raw blob contents from the repository' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 912415e3a7f..0d394a7b441 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -11,18 +11,18 @@ module API end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository tags' do - success Entities::RepoTag + success Entities::Tag end params do use :pagination end get ':id/repository/tags' do tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) - present paginate(tags), with: Entities::RepoTag, project: user_project + present paginate(tags), with: Entities::Tag, project: user_project end desc 'Get a single repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -31,11 +31,11 @@ module API tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag - present tag, with: Entities::RepoTag, project: user_project + present tag, with: Entities::Tag, project: user_project end desc 'Create a new repository tag' do - success Entities::RepoTag + success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' @@ -51,7 +51,7 @@ module API if result[:status] == :success present result[:tag], - with: Entities::RepoTag, + with: Entities::Tag, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index f70bc0622b7..6550b331fb8 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -49,7 +49,7 @@ module API desc 'Get the list of the available license template' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -60,12 +60,12 @@ module API featured: declared(params)[:popular].present? ? true : nil } licences = ::Kaminari.paginate_array(Licensee::License.all(options)) - present paginate(licences), with: Entities::RepoLicense + present paginate(licences), with: Entities::License end desc 'Get the text for a specific license' do detail 'This feature was introduced in GitLab 8.7.' - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -75,7 +75,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| diff --git a/lib/api/users.rb b/lib/api/users.rb index 1825c90a23b..b6f97a1eac2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,12 +6,14 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + include CustomAttributesEndpoints + before do authenticate_non_get! end helpers do - def find_user(params) + def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end @@ -88,7 +90,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) - opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : {} + opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User } present user, opts end @@ -166,7 +168,7 @@ module API user_params[:password_expires_at] = Time.now if user_params[:password].present? - result = ::Users::UpdateService.new(user, user_params.except(:extern_uid, :provider)).execute + result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute if result[:status] == :success present user, with: Entities::UserPublic @@ -326,10 +328,9 @@ module API user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - email = Emails::CreateService.new(user, declared_params(include_missing: false)).execute + email = Emails::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -367,10 +368,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: user).execute(email) end - - user.update_secondary_emails! end desc 'Delete a user. Available only for admins.' do @@ -430,7 +429,7 @@ module API resource :impersonation_tokens do helpers do def finder(options = {}) - user = find_user(params) + user = find_user_by_id(params) PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end @@ -672,10 +671,9 @@ module API requires :email, type: String, desc: 'The new email' end post "emails" do - email = Emails::CreateService.new(current_user, declared_params).execute + email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? - NotificationService.new.new_email(email) present email, with: Entities::Email else render_validation_error!(email) @@ -691,10 +689,8 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: current_user).execute(email) end - - current_user.update_secondary_emails! end desc 'Get a list of user activities' diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index 81b13249892..69cd12de72c 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -11,12 +11,12 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository branches' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) - present branches, with: ::API::Entities::RepoBranch, project: user_project + present branches, with: ::API::Entities::Branch, project: user_project end desc 'Delete a branch' @@ -47,7 +47,7 @@ module API end desc 'Create branch' do - success ::API::Entities::RepoBranch + success ::API::Entities::Branch end params do requires :branch_name, type: String, desc: 'The name of the branch' @@ -60,7 +60,7 @@ module API if result[:status] == :success present result[:branch], - with: ::API::Entities::RepoBranch, + with: ::API::Entities::Branch, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 5936f4700aa..345cb7e7c11 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -13,7 +13,7 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository commits' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -34,11 +34,11 @@ module API after: params[:since], before: params[:until]) - present commits, with: ::API::Entities::RepoCommit + present commits, with: ::API::Entities::Commit end desc 'Commit multiple file changes as one commit' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail detail 'This feature was introduced in GitLab 8.13' end params do @@ -59,14 +59,14 @@ module API if result[:status] == :success commit_detail = user_project.repository.commits(result[:result], limit: 1).first - present commit_detail, with: ::API::Entities::RepoCommitDetail + present commit_detail, with: ::API::Entities::CommitDetail else render_api_error!(result[:message], 400) end end desc 'Get a specific commit of a project' do - success ::API::Entities::RepoCommitDetail + success ::API::Entities::CommitDetail failure [[404, 'Not Found']] end params do @@ -77,7 +77,7 @@ module API not_found! "Commit" unless commit - present commit, with: ::API::Entities::RepoCommitDetail + present commit, with: ::API::Entities::CommitDetail end desc 'Get the diff for a specific commit of a project' do @@ -113,7 +113,7 @@ module API desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end params do requires :sha, type: String, desc: 'A commit sha to be cherry picked' @@ -138,7 +138,7 @@ module API if result[:status] == :success branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit + present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::Commit else render_api_error!(result[:message], 400) end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c928ce5265b..afdd7b83998 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -220,7 +220,7 @@ module API expose :created_at, :started_at, :finished_at expose :user, with: ::API::Entities::User expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } - expose :commit, with: ::API::Entities::RepoCommit + expose :commit, with: ::API::Entities::Commit expose :runner, with: ::API::Entities::Runner expose :pipeline, with: ::API::Entities::PipelineBasic end @@ -237,7 +237,7 @@ module API end class MergeRequestChanges < MergeRequest - expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| + expose :diffs, as: :changes, using: ::API::Entities::Diff do |compare, _| compare.raw_diffs(limits: false).to_a end end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index b6b7254ae29..1d6d823f32b 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -135,12 +135,12 @@ module API end desc 'Get the commits of a merge request' do - success ::API::Entities::RepoCommit + success ::API::Entities::Commit end get "#{path}/commits" do merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request.commits, with: ::API::Entities::RepoCommit + present merge_request.commits, with: ::API::Entities::Commit end desc 'Show the merge request changes' do diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index 0eaa0de2eef..41a7c6b83ae 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -19,7 +19,7 @@ module API end desc 'Get a project repository tree' do - success ::API::Entities::RepoTreeObject + success ::API::Entities::TreeObject end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' @@ -35,7 +35,7 @@ module API tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - present tree.sorted_entries, with: ::API::Entities::RepoTreeObject + present tree.sorted_entries, with: ::API::Entities::TreeObject end desc 'Get a raw file contents' diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb index 7e5875cd030..6e37d31d153 100644 --- a/lib/api/v3/tags.rb +++ b/lib/api/v3/tags.rb @@ -8,11 +8,11 @@ module API end resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository tags' do - success ::API::Entities::RepoTag + success ::API::Entities::Tag end get ":id/repository/tags" do tags = user_project.repository.tags.sort_by(&:name).reverse - present tags, with: ::API::Entities::RepoTag, project: user_project + present tags, with: ::API::Entities::Tag, project: user_project end desc 'Delete a repository tag' diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 2a2fb59045c..7298203df10 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -52,7 +52,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' @@ -61,7 +61,7 @@ module API options = { featured: declared(params)[:popular].present? ? true : nil } - present Licensee::License.all(options), with: ::API::Entities::RepoLicense + present Licensee::License.all(options), with: ::API::Entities::License end end @@ -70,7 +70,7 @@ module API detailed_desc = 'This feature was introduced in GitLab 8.7.' detailed_desc << DEPRECATION_MESSAGE unless status == :ok detail detailed_desc - success ::API::Entities::RepoLicense + success ::API::Entities::License end params do requires :name, type: String, desc: 'The name of the template' @@ -80,7 +80,7 @@ module API template = parsed_license_template - present template, with: ::API::Entities::RepoLicense + present template, with: ::API::Entities::License end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index b9a573d3542..3cf3939994a 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -79,7 +79,7 @@ module Backup # - 1495527122_gitlab_backup.tar # - 1495527068_2017_05_23_gitlab_backup.tar # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar - next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/ + next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/ timestamp = $1.to_i diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4e92be85110..3ad09a1b421 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -78,7 +78,7 @@ module Backup project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) else %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index bcb4f332267..4cd9b02b76c 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -1,6 +1,7 @@ module Banzai module Filter - # HTML filter that moves the value of the src attribute to the data-src attribute so it can be lazy loaded + # HTML filter that moves the value of image `src` attributes to `data-src` + # so they can be lazy loaded. class ImageLazyLoadFilter < HTML::Pipeline::Filter def call doc.xpath('descendant-or-self::img').each do |img| diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ee73fa91589..9cac303e645 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,6 +1,18 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + REDCARPET_OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + def initialize(text, context = nil, result = nil) super text, context, result @text = @text.delete "\r" @@ -13,27 +25,11 @@ module Banzai end def self.renderer - @renderer ||= begin + Thread.current[:banzai_markdown_renderer] ||= begin renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) end end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - private_class_method :redcarpet_options end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 2d6e8ffc90f..d8c8deea628 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -5,6 +5,7 @@ module Banzai # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ def whitelist whitelist = super @@ -24,7 +25,8 @@ module Banzai # Only push these customizations once return if customized?(whitelist[:transformers]) - # Allow table alignment + # Allow table alignment; we whitelist specific style properties in a + # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) @@ -43,6 +45,10 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) + # Disallow `name` attribute globally, allow on `a` + whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') + # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') @@ -52,6 +58,9 @@ module Banzai # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) + # Remove any `style` properties not required for table alignment + whitelist[:transformers].push(self.class.remove_unsafe_table_style) + whitelist end @@ -64,8 +73,9 @@ module Banzai return unless node.has_attribute?('href') begin + node['href'] = node['href'].strip uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + uri.scheme = uri.scheme.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError @@ -81,6 +91,21 @@ module Banzai end end end + + def remove_unsafe_table_style + lambda do |env| + node = env[:node] + + return unless node.name == 'th' || node.name == 'td' + return unless node.has_attribute?('style') + + if node['style'] =~ TABLE_ALIGNMENT_PATTERN + node['style'] = "text-align: #{$~[:alignment]}" + else + node.remove_attribute('style') + end + end + end end end end diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index e47c384afc1..8f5f144d582 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,6 +1,12 @@ module Banzai module Pipeline class EmailPipeline < FullPipeline + def self.filters + super.tap do |filter_array| + filter_array.delete(Banzai::Filter::ImageLazyLoadFilter) + end + end + def self.transform_context(context) super(context).merge( only_path: false diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index a65bbe23958..e0a8ca653cb 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -34,7 +34,8 @@ module Banzai { namespace: :owner }, { group: [:owners, :group_members] }, :invited_groups, - :project_members + :project_members, + :project_feature ] } ), diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ceca9296851..5f91884a878 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -40,7 +40,7 @@ module Banzai return cacheless_render_field(object, field) end - object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) + object.refresh_markdown_cache! unless object.cached_html_up_to_date?(field) object.cached_html_for(field) end @@ -162,10 +162,5 @@ module Banzai return unless cache_key Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end - - # GitLab EE needs to disable updates on GET requests in Geo - def self.update_object?(object) - true - end end end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb deleted file mode 100644 index b9e9f9f7f4a..00000000000 --- a/lib/ci/ansi2html.rb +++ /dev/null @@ -1,331 +0,0 @@ -# ANSI color library -# -# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code -module Ci - module Ansi2html - # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) - COLOR = { - 0 => 'black', # not that this is gray in the intense color table - 1 => 'red', - 2 => 'green', - 3 => 'yellow', - 4 => 'blue', - 5 => 'magenta', - 6 => 'cyan', - 7 => 'white', # not that this is gray in the dark (aka default) color table - }.freeze - - STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 - }.freeze - - def self.convert(ansi, state = nil) - Converter.new.convert(ansi, state) - end - - class Converter - def on_0(s) reset() end - - def on_1(s) enable(STYLE_SWITCHES[:bold]) end - - def on_3(s) enable(STYLE_SWITCHES[:italic]) end - - def on_4(s) enable(STYLE_SWITCHES[:underline]) end - - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end - - def on_9(s) enable(STYLE_SWITCHES[:cross]) end - - def on_21(s) disable(STYLE_SWITCHES[:bold]) end - - def on_22(s) disable(STYLE_SWITCHES[:bold]) end - - def on_23(s) disable(STYLE_SWITCHES[:italic]) end - - def on_24(s) disable(STYLE_SWITCHES[:underline]) end - - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end - - def on_29(s) disable(STYLE_SWITCHES[:cross]) end - - def on_30(s) set_fg_color(0) end - - def on_31(s) set_fg_color(1) end - - def on_32(s) set_fg_color(2) end - - def on_33(s) set_fg_color(3) end - - def on_34(s) set_fg_color(4) end - - def on_35(s) set_fg_color(5) end - - def on_36(s) set_fg_color(6) end - - def on_37(s) set_fg_color(7) end - - def on_38(s) set_fg_color_256(s) end - - def on_39(s) set_fg_color(9) end - - def on_40(s) set_bg_color(0) end - - def on_41(s) set_bg_color(1) end - - def on_42(s) set_bg_color(2) end - - def on_43(s) set_bg_color(3) end - - def on_44(s) set_bg_color(4) end - - def on_45(s) set_bg_color(5) end - - def on_46(s) set_bg_color(6) end - - def on_47(s) set_bg_color(7) end - - def on_48(s) set_bg_color_256(s) end - - def on_49(s) set_bg_color(9) end - - def on_90(s) set_fg_color(0, 'l') end - - def on_91(s) set_fg_color(1, 'l') end - - def on_92(s) set_fg_color(2, 'l') end - - def on_93(s) set_fg_color(3, 'l') end - - def on_94(s) set_fg_color(4, 'l') end - - def on_95(s) set_fg_color(5, 'l') end - - def on_96(s) set_fg_color(6, 'l') end - - def on_97(s) set_fg_color(7, 'l') end - - def on_99(s) set_fg_color(9, 'l') end - - def on_100(s) set_bg_color(0, 'l') end - - def on_101(s) set_bg_color(1, 'l') end - - def on_102(s) set_bg_color(2, 'l') end - - def on_103(s) set_bg_color(3, 'l') end - - def on_104(s) set_bg_color(4, 'l') end - - def on_105(s) set_bg_color(5, 'l') end - - def on_106(s) set_bg_color(6, 'l') end - - def on_107(s) set_bg_color(7, 'l') end - - def on_109(s) set_bg_color(9, 'l') end - - attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - - def convert(stream, new_state) - reset_state - restore_state(new_state, stream) if new_state.present? - - append = false - truncated = false - - cur_offset = stream.tell - if cur_offset > @offset - @offset = cur_offset - truncated = true - else - stream.seek(@offset) - append = @offset > 0 - end - start_offset = @offset - - open_new_tag - - stream.each_line do |line| - s = StringScanner.new(line) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) - end - @offset += s.matched_size - end - end - - close_open_tags() - - OpenStruct.new( - html: @out.force_encoding(Encoding.default_external), - state: state, - append: append, - truncated: truncated, - offset: start_offset, - size: stream.tell - start_offset, - total: stream.size - ) - end - - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] - - # We are only interested in color and text style changes - triggered by - # sequences starting with '\e[' and ending with 'm'. Any other control - # sequence gets stripped (including stuff like "delete last line") - return unless indicator == '[' && terminator == 'm' - - close_open_tags() - - if commands.empty?() - reset() - return - end - - evaluate_command_stack(commands) - - open_new_tag - end - - def evaluate_command_stack(stack) - return unless command = stack.shift() - - if self.respond_to?("on_#{command}", true) - self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend - end - - evaluate_command_stack(stack) - end - - def open_new_tag - css_classes = [] - - unless @fg_color.nil? - fg_color = @fg_color - # Most terminals show bold colored text in the light color variant - # Let's mimic that here - if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') - end - css_classes << fg_color - end - css_classes << @bg_color unless @bg_color.nil? - - STYLE_SWITCHES.each do |css_class, flag| - css_classes << "term-#{css_class}" if @style_mask & flag != 0 - end - - return if css_classes.empty? - - @out << %{<span class="#{css_classes.join(' ')}">} - @n_open_tags += 1 - end - - def close_open_tags - while @n_open_tags > 0 - @out << %{</span>} - @n_open_tags -= 1 - end - end - - def reset_state - @offset = 0 - @n_open_tags = 0 - @out = '' - reset - end - - def state - state = STATE_PARAMS.inject({}) do |h, param| - h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend - h - end - Base64.urlsafe_encode64(state.to_json) - end - - def restore_state(new_state, stream) - state = Base64.urlsafe_decode64(new_state) - state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > stream.size - - STATE_PARAMS.each do |param| - send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def reset - @fg_color = nil - @bg_color = nil - @style_mask = 0 - end - - def enable(flag) - @style_mask |= flag - end - - def disable(flag) - @style_mask &= ~flag - end - - def set_fg_color(color_index, prefix = nil) - @fg_color = get_term_color_class(color_index, ["fg", prefix]) - end - - def set_bg_color(color_index, prefix = nil) - @bg_color = get_term_color_class(color_index, ["bg", prefix]) - end - - def get_term_color_class(color_index, prefix) - color_name = COLOR[color_index] - return nil if color_name.nil? - - get_color_class(["term", prefix, color_name]) - end - - def set_fg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "fg") - @fg_color = css_class unless css_class.nil? - end - - def set_bg_color_256(command_stack) - css_class = get_xterm_color_class(command_stack, "bg") - @bg_color = css_class unless css_class.nil? - end - - def get_xterm_color_class(command_stack, prefix) - # the 38 and 48 commands have to be followed by "5" and the color index - return unless command_stack.length >= 2 - return unless command_stack[0] == "5" - - command_stack.shift() # ignore the "5" command - color_index = command_stack.shift().to_i - - return unless color_index >= 0 - return unless color_index <= 255 - - get_color_class(["xterm", prefix, color_index]) - end - - def get_color_class(segments) - [segments].flatten.compact.join('-') - end - end - end -end diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/ci/assets/.gitkeep +++ /dev/null diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb deleted file mode 100644 index 76a69bf8a83..00000000000 --- a/lib/ci/charts.rb +++ /dev/null @@ -1,116 +0,0 @@ -module Ci - module Charts - module DailyInterval - def grouped_count(query) - query - .group("DATE(#{Ci::Pipeline.table_name}.created_at)") - .count(:created_at) - .transform_keys { |date| date.strftime(@format) } - end - - def interval_step - @interval_step ||= 1.day - end - end - - module MonthlyInterval - def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end - end - - def interval_step - @interval_step ||= 1.month - end - end - - class Chart - attr_reader :labels, :total, :success, :project, :pipeline_times - - def initialize(project) - @labels = [] - @total = [] - @success = [] - @pipeline_times = [] - @project = project - - collect - end - - def collect - query = project.pipelines - .where("? > #{Ci::Pipeline.table_name}.created_at AND #{Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection - - totals_count = grouped_count(query) - success_count = grouped_count(query.success) - - current = @from - while current < @to - label = current.strftime(@format) - - @labels << label - @total << (totals_count[label] || 0) - @success << (success_count[label] || 0) - - current += interval_step - end - end - end - - class YearChart < Chart - include MonthlyInterval - - def initialize(*) - @to = Date.today.end_of_month - @from = @to.years_ago(1).beginning_of_month - @format = '%d %B %Y' - - super - end - end - - class MonthChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 30.days - @format = '%d %B' - - super - end - end - - class WeekChart < Chart - include DailyInterval - - def initialize(*) - @to = Date.today - @from = @to - 7.days - @format = '%d %B' - - super - end - end - - class PipelineTime < Chart - def collect - commits = project.pipelines.last(30) - - commits.each do |commit| - @labels << commit.short_sha - duration = commit.duration || 0 - @pipeline_times << (duration / 60) - end - end - end - end -end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb deleted file mode 100644 index 62b44389b15..00000000000 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ /dev/null @@ -1,251 +0,0 @@ -module Ci - class GitlabCiYamlProcessor - ValidationError = Class.new(StandardError) - - include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - - attr_reader :path, :cache, :stages, :jobs - - def initialize(config, path = nil) - @ci_config = Gitlab::Ci::Config.new(config) - @config = @ci_config.to_hash - @path = path - - unless @ci_config.valid? - raise ValidationError, @ci_config.errors.first - end - - initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e - raise ValidationError, e.message - end - - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| - build_attributes(name) - end - end - - def builds - @jobs.map do |name, _| - build_attributes(name) - end - end - - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) - - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end - - seeds.compact - end - - def build_attributes(name) - job = @jobs[name.to_sym] || {} - - { stage_idx: @stages.index(job[:stage]), - stage: job[:stage], - commands: job[:commands], - tag_list: job[:tags] || [], - name: job[:name].to_s, - allow_failure: job[:ignore], - when: job[:when] || 'on_success', - environment: job[:environment_name], - coverage_regex: job[:coverage], - yaml_variables: yaml_variables(name), - options: { - image: job[:image], - services: job[:services], - artifacts: job[:artifacts], - cache: job[:cache], - dependencies: job[:dependencies], - before_script: job[:before_script], - script: job[:script], - after_script: job[:after_script], - environment: job[:environment], - retry: job[:retry] - }.compact } - end - - def self.validation_message(content) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Ci::GitlabCiYamlProcessor.new(content) - nil - rescue ValidationError, Psych::SyntaxError => e - e.message - end - end - - private - - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_active? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def initial_parsing - ## - # Global config - # - @before_script = @ci_config.before_script - @image = @ci_config.image - @after_script = @ci_config.after_script - @services = @ci_config.services - @variables = @ci_config.variables - @stages = @ci_config.stages - @cache = @ci_config.cache - - ## - # Jobs - # - @jobs = @ci_config.jobs - - @jobs.each do |name, job| - # logical validation for job - - validate_job_stage!(name, job) - validate_job_dependencies!(name, job) - validate_job_environment!(name, job) - end - end - - def yaml_variables(name) - variables = (@variables || {}) - .merge(job_variables(name)) - - variables.map do |key, value| - { key: key.to_s, value: value, public: true } - end - end - - def job_variables(name) - job = @jobs[name.to_sym] - return {} unless job - - job[:variables] || {} - end - - def validate_job_stage!(name, job) - return unless job[:stage] - - unless job[:stage].is_a?(String) && job[:stage].in?(@stages) - raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" - end - end - - def validate_job_dependencies!(name, job) - return unless job[:dependencies] - - stage_index = @stages.index(job[:stage]) - - job[:dependencies].each do |dependency| - raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index - raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" - end - end - end - - def validate_job_environment!(name, job) - return unless job[:environment] - return unless job[:environment].is_a?(Hash) - - environment = job[:environment] - validate_on_stop_job!(name, environment, environment[:on_stop]) - end - - def validate_on_stop_job!(name, environment, on_stop) - return unless on_stop - - on_stop_job = @jobs[on_stop.to_sym] - unless on_stop_job - raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" - end - - unless on_stop_job[:environment] - raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" - end - - unless on_stop_job[:environment][:name] == environment[:name] - raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" - end - - unless on_stop_job[:environment][:action] == 'stop' - raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" - end - end - - def process?(only_params, except_params, ref, tag, source) - if only_params.present? - return false unless matching?(only_params, ref, tag, source) - end - - if except_params.present? - return false if matching?(except_params, ref, tag, source) - end - - true - end - - def matching?(patterns, ref, tag, source) - patterns.any? do |pattern| - pattern, path = pattern.split('@', 2) - matches_path?(path) && matches_pattern?(pattern, ref, tag, source) - end - end - - def matches_path?(path) - return true unless path - - path == self.path - end - - def matches_pattern?(pattern, ref, tag, source) - return true if tag && pattern == 'tags' - return true if !tag && pattern == 'branches' - return true if source_to_pattern(source) == pattern - - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end - end -end diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb deleted file mode 100644 index 997377abc55..00000000000 --- a/lib/ci/mask_secret.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Ci::MaskSecret - class << self - def mask!(value, token) - return value unless value.present? && token.present? - - value.gsub!(token, 'x' * token.length) - value - end - end -end diff --git a/lib/ci/model.rb b/lib/ci/model.rb deleted file mode 100644 index c42a0ad36db..00000000000 --- a/lib/ci/model.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Ci - module Model - def table_name_prefix - "ci_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) - end - end -end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index bfcec241489..7cfa82a9a9f 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -206,11 +206,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return false if passes.any? { |p| p == false } - return true if passes.all? { |p| p == true } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == false + end + + true end def repr @@ -245,11 +247,13 @@ module DeclarativePolicy end def cached_pass?(context) - passes = @rules.map { |r| r.cached_pass?(context) } - return true if passes.any? { |p| p == true } - return false if passes.all? { |p| p == false } + @rules.each do |rule| + pass = rule.cached_pass?(context) - nil + return pass if pass.nil? || pass == true + end + + false end def score(context) diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 56afd1f1392..45ff2ef9ced 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -107,7 +107,7 @@ module DeclarativePolicy end # This is the core spot where all those `#score` methods matter. - # It is critcal for performance to run steps in the correct order, + # It is critical for performance to run steps in the correct order, # so that we don't compute expensive conditions (potentially n times # if we're called on, say, a large list of users). # @@ -139,30 +139,39 @@ module DeclarativePolicy return end - steps = Set.new(@steps) - remaining_enablers = steps.count { |s| s.enable? } + remaining_steps = Set.new(@steps) + remaining_enablers, remaining_preventers = remaining_steps.partition(&:enable?).map { |s| Set.new(s) } loop do - return if steps.empty? + if @state.enabled? + # Once we set this, we never need to unset it, because a single + # prevent will stop this from being enabled + remaining_steps = remaining_preventers + else + # if the permission hasn't yet been enabled and we only have + # prevent steps left, we short-circuit the state here + @state.prevent! if remaining_enablers.empty? + end - # if the permission hasn't yet been enabled and we only have - # prevent steps left, we short-circuit the state here - @state.prevent! if !@state.enabled? && remaining_enablers == 0 + return if remaining_steps.empty? lowest_score = Float::INFINITY next_step = nil - steps.each do |step| + remaining_steps.each do |step| score = step.score + if score < lowest_score next_step = step lowest_score = score end - end - steps.delete(next_step) + break if lowest_score.zero? + end - remaining_enablers -= 1 if next_step.enable? + [remaining_steps, remaining_enablers, remaining_preventers].each do |set| + set.delete(next_step) + end yield next_step, lowest_score end diff --git a/lib/github/client.rb b/lib/github/client.rb index 9c476df7d46..29bd9c1f39e 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,6 +1,7 @@ module Github class Client TIMEOUT = 60 + DEFAULT_PER_PAGE = 100 attr_reader :connection, :rate_limit @@ -20,7 +21,7 @@ module Github exceed, reset_in = rate_limit.get sleep reset_in if exceed - Github::Response.new(connection.get(url, query)) + Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) end private diff --git a/lib/github/import.rb b/lib/github/import.rb index 9354e142d3d..55f8387f27a 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,48 +1,15 @@ require_relative 'error' +require_relative 'import/issue' +require_relative 'import/legacy_diff_note' +require_relative 'import/merge_request' +require_relative 'import/note' module Github class Import include Gitlab::ShellAdapter - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Note < ::Note - self.table_name = 'notes' - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose + :options, :errors, :cached, :verbose, :last_fetched_at def initialize(project, options = {}) @project = project @@ -54,12 +21,13 @@ module Github @verbose = options.fetch(:verbose, false) @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] + @last_fetched_at = nil end # rubocop: disable Rails/Output def execute puts 'Fetching repository...'.color(:aqua) if verbose - fetch_repository + setup_and_fetch_repository puts 'Fetching labels...'.color(:aqua) if verbose fetch_labels puts 'Fetching milestones...'.color(:aqua) if verbose @@ -75,7 +43,7 @@ module Github puts 'Expiring repository cache...'.color(:aqua) if verbose expire_repository_cache - true + errors.empty? rescue Github::RepositoryFetchError expire_repository_cache false @@ -85,18 +53,24 @@ module Github private - def fetch_repository + def setup_and_fetch_repository begin project.ensure_repository project.repository.add_remote('github', repo_url) - project.repository.set_remote_as_mirror('github') - project.repository.fetch_remote('github', forced: true) + project.repository.set_import_remote_as_mirror('github') + project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') + fetch_remote(forced: true) rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e error(:project, repo_url, e.message) raise Github::RepositoryFetchError end end + def fetch_remote(forced: false) + @last_fetched_at = Time.now + project.repository.fetch_remote('github', forced: forced) + end + def fetch_wiki_repository return if project.wiki.repository_exists? @@ -125,7 +99,7 @@ module Github label.color = representation.color end - cached[:label_ids][label.title] = label.id + cached[:label_ids][representation.title] = label.id rescue => e error(:label, representation.url, e.message) end @@ -176,7 +150,9 @@ module Github next unless merge_request.new_record? && pull_request.valid? begin - pull_request.restore_branches! + # If the PR has been created/updated after we last fetched the + # remote, we fetch again to get the up-to-date refs. + fetch_remote if pull_request.updated_at > last_fetched_at author_id = user_id(pull_request.author, project.creator_id) description = format_description(pull_request.description, pull_request.author) @@ -185,6 +161,7 @@ module Github iid: pull_request.iid, title: pull_request.title, description: description, + ref_fetched: true, source_project: pull_request.source_project, source_branch: pull_request.source_branch_name, source_branch_sha: pull_request.source_branch_sha, @@ -202,17 +179,10 @@ module Github merge_request.save!(validate: false) merge_request.merge_request_diffs.create - # Fetch review comments review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - - # Fetch comments - comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" - fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) - ensure - pull_request.remove_restored_branches! end end @@ -241,12 +211,17 @@ module Github # for both features, like manipulating assignees, labels # and milestones, are provided within the Issues API. if representation.pull_request? - return unless representation.has_labels? + return unless representation.labels? || representation.comments? merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + + if representation.labels? + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + end + + fetch_comments_conditionally(merge_request, representation) else - return if Issue.where(iid: representation.iid, project_id: project.id).exists? + return if Issue.exists?(iid: representation.iid, project_id: project.id) author_id = user_id(representation.author, project.creator_id) issue = Issue.new @@ -255,25 +230,30 @@ module Github issue.title = representation.title issue.description = format_description(representation.description, representation.author) issue.state = representation.state - issue.label_ids = label_ids(representation.labels) issue.milestone_id = milestone_id(representation.milestone) issue.author_id = author_id - issue.assignee_ids = [user_id(representation.assignee)] issue.created_at = representation.created_at issue.updated_at = representation.updated_at issue.save!(validate: false) - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end + issue.update( + label_ids: label_ids(representation.labels), + assignee_ids: assignee_ids(representation.assignees)) + + fetch_comments_conditionally(issue, representation) end rescue => e error(:issue, representation.url, e.message) end end + def fetch_comments_conditionally(issuable, representation) + if representation.comments? + comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" + fetch_comments(issuable, :comment, comments_url) + end + end + def fetch_comments(noteable, type, url, klass = Note) while url comments = Github::Client.new(options).get(url) @@ -332,7 +312,11 @@ module Github end def label_ids(labels) - labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + labels.map { |label| cached[:label_ids][label.title] }.compact + end + + def assignee_ids(assignees) + assignees.map { |assignee| user_id(assignee) }.compact end def milestone_id(milestone) diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb new file mode 100644 index 00000000000..171f0872666 --- /dev/null +++ b/lib/github/import/issue.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb new file mode 100644 index 00000000000..18adff560b6 --- /dev/null +++ b/lib/github/import/legacy_diff_note.rb @@ -0,0 +1,12 @@ +module Github + class Import + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb new file mode 100644 index 00000000000..c258e5d5e0e --- /dev/null +++ b/lib/github/import/merge_request.rb @@ -0,0 +1,13 @@ +module Github + class Import + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb new file mode 100644 index 00000000000..8cf4f30e6b7 --- /dev/null +++ b/lib/github/import/note.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Note < ::Note + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index 823e8e9a9c4..0087a3d3c4f 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -7,10 +7,14 @@ module Github raw.dig('user', 'login') || 'unknown' end + def repo? + raw['repo'].present? + end + def repo - return @repo if defined?(@repo) + return unless repo? - @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + @repo ||= Github::Representation::Repo.new(raw['repo']) end def ref @@ -25,10 +29,6 @@ module Github Commit.truncate_sha(sha) end - def exists? - @exists ||= branch_exists? && commit_exists? - end - def valid? sha.present? && ref.present? end @@ -47,14 +47,6 @@ module Github private - def branch_exists? - repository.branch_exists?(ref) - end - - def commit_exists? - repository.branch_names_contains(sha).include?(ref) - end - def repository @repository ||= options.fetch(:repository) end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb index 9713b82615d..768ba3b993c 100644 --- a/lib/github/representation/issuable.rb +++ b/lib/github/representation/issuable.rb @@ -23,14 +23,14 @@ module Github @author ||= Github::Representation::User.new(raw['user'], options) end - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) + def labels? + raw['labels'].any? end - def assigned? - raw['assignee'].present? + def labels + @labels ||= Array(raw['labels']).map do |label| + Github::Representation::Label.new(label, options) + end end end end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index df3540a6e6c..4f1a02cb90f 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -1,25 +1,27 @@ module Github module Representation class Issue < Representation::Issuable - def labels - raw['labels'] - end - def state raw['state'] == 'closed' ? 'closed' : 'opened' end - def has_comments? + def comments? raw['comments'] > 0 end - def has_labels? - labels.count > 0 - end - def pull_request? raw['pull_request'].present? end + + def assigned? + raw['assignees'].present? + end + + def assignees + @assignees ||= Array(raw['assignees']).map do |user| + Github::Representation::User.new(user, options) + end + end end end end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 55461097e8a..0171179bb0f 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -1,26 +1,17 @@ module Github module Representation class PullRequest < Representation::Issuable - delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true - delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + delegate :sha, to: :source_branch, prefix: true + delegate :sha, to: :target_branch, prefix: true def source_project project end def source_branch_name - @source_branch_name ||= - if cross_project? || !source_branch_exists? - source_branch_name_prefixed - else - source_branch_ref - end - end - - def source_branch_exists? - return @source_branch_exists if defined?(@source_branch_exists) - - @source_branch_exists = !cross_project? && source_branch.exists? + # Mimic the "user:branch" displayed in the MR widget, + # i.e. "Request to merge rymai:add-external-mounts into master" + cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref end def target_project @@ -28,11 +19,7 @@ module Github end def target_branch_name - @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed - end - - def target_branch_exists? - @target_branch_exists ||= target_branch.exists? + target_branch.ref end def state @@ -50,16 +37,14 @@ module Github source_branch.valid? && target_branch.valid? end - def restore_branches! - restore_source_branch! - restore_target_branch! + def assigned? + raw['assignee'].present? end - def remove_restored_branches! - return if opened? + def assignee + return unless assigned? - remove_source_branch! - remove_target_branch! + @assignee ||= Github::Representation::User.new(raw['assignee'], options) end private @@ -72,48 +57,14 @@ module Github @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) end - def source_branch_name_prefixed - "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" - end - def target_branch @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) end - def target_branch_name_prefixed - "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" - end - def cross_project? - return true if source_branch_repo.nil? - - source_branch_repo.id != target_branch_repo.id - end - - def restore_source_branch! - return if source_branch_exists? - - source_branch.restore!(source_branch_name) - end - - def restore_target_branch! - return if target_branch_exists? - - target_branch.restore!(target_branch_name) - end - - def remove_source_branch! - # We should remove the source/target branches only if they were - # restored. Otherwise, we'll remove branches like 'master' that - # target_branch_exists? returns true. In other words, we need - # to clean up only the restored branches that (source|target)_branch_exists? - # returns false for the first time it has been called, because of - # this that is important to memoize these values. - source_branch.remove!(source_branch_name) unless source_branch_exists? - end + return true unless source_branch.repo? - def remove_target_branch! - target_branch.remove!(target_branch_name) unless target_branch_exists? + source_branch.repo.id != target_branch.repo.id end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3fd81759d25..87aeb76b66a 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -13,11 +13,6 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze - AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze - - # Other available scopes - OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze - class << self include Gitlab::CurrentSettings @@ -132,7 +127,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_scoped_token?(token, AVAILABLE_SCOPES) + if token && valid_scoped_token?(token, available_scopes) Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end @@ -230,6 +225,21 @@ module Gitlab def read_user_scope_authentication_abilities [] end + + def available_scopes + API_SCOPES + registry_scopes + end + + # Other available scopes + def optional_scopes + available_scopes + OPENID_SCOPES - DEFAULT_SCOPES + end + + def registry_scopes + return [] unless Gitlab.config.registry.enabled + + REGISTRY_SCOPES + end end end end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb new file mode 100644 index 00000000000..c88eb9783ed --- /dev/null +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -0,0 +1,65 @@ +module Gitlab + module BackgroundMigration + class CreateForkNetworkMembershipsRange + RESCHEDULE_DELAY = 15 + + class ForkedProjectLink < ActiveRecord::Base + self.table_name = 'forked_project_links' + end + + def perform(start_id, end_id) + log("Creating memberships for forks: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS + INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) + + SELECT fork_network_members.fork_network_id, + forked_project_links.forked_to_project_id, + forked_project_links.forked_from_project_id + + FROM forked_project_links + + INNER JOIN fork_network_members + ON forked_project_links.forked_from_project_id = fork_network_members.project_id + + WHERE forked_project_links.id BETWEEN #{start_id} AND #{end_id} + AND NOT EXISTS ( + SELECT true + FROM fork_network_members existing_members + WHERE existing_members.project_id = forked_project_links.forked_to_project_id + ) + INSERT_MEMBERS + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def missing_members?(start_id, end_id) + count_sql = <<~MISSING_MEMBERS + SELECT COUNT(*) + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = forked_project_links.forked_to_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE forked_project_links.forked_from_project_id = projects.id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + MISSING_MEMBERS + + ForkNetworkMember.count_by_sql(count_sql) > 0 + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb new file mode 100644 index 00000000000..e94719db72e --- /dev/null +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -0,0 +1,53 @@ +class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys + class GpgKey < ActiveRecord::Base + self.table_name = 'gpg_keys' + + include EachBatch + include ShaAttribute + + sha_attribute :primary_keyid + sha_attribute :fingerprint + + has_many :subkeys, class_name: 'GpgKeySubkey' + end + + class GpgKeySubkey < ActiveRecord::Base + self.table_name = 'gpg_key_subkeys' + + include ShaAttribute + + sha_attribute :keyid + sha_attribute :fingerprint + end + + def perform(gpg_key_id) + gpg_key = GpgKey.find_by(id: gpg_key_id) + + return if gpg_key.nil? + return if gpg_key.subkeys.any? + + create_subkeys(gpg_key) + update_signatures(gpg_key) + end + + private + + def create_subkeys(gpg_key) + gpg_subkeys = Gitlab::Gpg.subkeys_from_key(gpg_key.key) + + gpg_subkeys[gpg_key.primary_keyid.upcase]&.each do |subkey_data| + gpg_key.subkeys.build(keyid: subkey_data[:keyid], fingerprint: subkey_data[:fingerprint]) + end + + # Improve latency by doing all INSERTs in a single call + GpgKey.transaction do + gpg_key.save! + end + end + + def update_signatures(gpg_key) + return unless gpg_key.subkeys.exists? + + InvalidGpgSignatureUpdateWorker.perform_async(gpg_key.id) + end +end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb new file mode 100644 index 00000000000..b1411be3016 --- /dev/null +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -0,0 +1,41 @@ +module Gitlab + module BackgroundMigration + class DeleteConflictingRedirectRoutesRange + class Route < ActiveRecord::Base + self.table_name = 'routes' + end + + class RedirectRoute < ActiveRecord::Base + self.table_name = 'redirect_routes' + end + + # start_id - The start ID of the range of events to process + # end_id - The end ID of the range to process. + def perform(start_id, end_id) + return unless migrate? + + conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id)) + num_rows = conflicts.delete_all + + Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.") + end + + def migrate? + Route.table_exists? && RedirectRoute.table_exists? + end + + def routes_match_redirects_clause(start_id, end_id) + <<~ROUTES_MATCH_REDIRECTS + EXISTS ( + SELECT 1 FROM routes + WHERE ( + LOWER(redirect_routes.path) = LOWER(routes.path) + OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%')) + ) + AND routes.id BETWEEN #{start_id} AND #{end_id} + ) + ROUTES_MATCH_REDIRECTS + end + end + end +end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 3fde1b09efb..8e5c95f2287 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -3,11 +3,18 @@ module Gitlab class DeserializeMergeRequestDiffsAndCommits attr_reader :diff_ids, :commit_rows, :file_rows + class Error < StandardError + def backtrace + cause.backtrace + end + end + class MergeRequestDiff < ActiveRecord::Base self.table_name = 'merge_request_diffs' end BUFFER_ROWS = 1000 + DIFF_FILE_BUFFER_ROWS = 100 def perform(start_id, stop_id) merge_request_diffs = MergeRequestDiff @@ -26,13 +33,17 @@ module Gitlab if diff_ids.length > BUFFER_ROWS || commit_rows.length > BUFFER_ROWS || - file_rows.length > BUFFER_ROWS + file_rows.length > DIFF_FILE_BUFFER_ROWS flush_buffers! end end flush_buffers! + rescue => e + Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") + + raise Error.new(e.inspect) end private @@ -45,17 +56,28 @@ module Gitlab def flush_buffers! if diff_ids.any? - MergeRequestDiff.transaction do - Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) - Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| + bulk_insert('merge_request_diff_commits', commit_rows_slice) + end - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| + bulk_insert('merge_request_diff_files', file_rows_slice) end + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) end reset_buffers! end + def bulk_insert(table, rows) + Gitlab::Database.bulk_insert(table, rows) + rescue ActiveRecord::RecordNotUnique + ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort + + Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") + end + def single_diff_rows(merge_request_diff) sha_attribute = Gitlab::Database::ShaAttribute.new commits = YAML.load(merge_request_diff.st_commits) rescue [] diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb new file mode 100644 index 00000000000..bc53e6d7f94 --- /dev/null +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -0,0 +1,313 @@ +module Gitlab + module BackgroundMigration + class NormalizeLdapExternUidsRange + class Identity < ActiveRecord::Base + self.table_name = 'identities' + end + + # Copied this class to make this migration resilient to future code changes. + # And if the normalize behavior is changed in the future, it must be + # accompanied by another migration. + module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end + end + + def perform(start_id, end_id) + return unless migrate? + + ldap_identities = Identity.where("provider like 'ldap%'").where(id: start_id..end_id) + ldap_identities.each do |identity| + begin + identity.extern_uid = Gitlab::LDAP::DN.new(identity.extern_uid).to_normalized_s + unless identity.save + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\". Skipping." + end + rescue Gitlab::LDAP::DN::FormatError => e + Rails.logger.info "Unable to normalize \"#{identity.extern_uid}\" due to \"#{e.message}\". Skipping." + end + end + end + + def migrate? + Identity.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb new file mode 100644 index 00000000000..2ef3a207dd3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -0,0 +1,59 @@ +module Gitlab + module BackgroundMigration + class PopulateForkNetworksRange + def perform(start_id, end_id) + log("Creating fork networks for forked project links: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS + INSERT INTO fork_networks (root_project_id) + SELECT DISTINCT forked_project_links.forked_from_project_id + + FROM forked_project_links + + WHERE NOT EXISTS ( + SELECT true + FROM forked_project_links inner_links + WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id + ) + AND NOT EXISTS ( + SELECT true + FROM fork_networks + WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id + ) + AND EXISTS ( + SELECT true + FROM projects + WHERE projects.id = forked_project_links.forked_from_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_NETWORKS + + log("Creating memberships for root projects: #{start_id} - #{end_id}") + + ActiveRecord::Base.connection.execute <<~INSERT_ROOT + INSERT INTO fork_network_members (fork_network_id, project_id) + SELECT DISTINCT fork_networks.id, fork_networks.root_project_id + + FROM fork_networks + + INNER JOIN forked_project_links + ON forked_project_links.forked_from_project_id = fork_networks.root_project_id + + WHERE NOT EXISTS ( + SELECT true + FROM fork_network_members + WHERE fork_network_members.project_id = fork_networks.root_project_id + ) + AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} + INSERT_ROOT + + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY + BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + + def log(message) + Rails.logger.info("#{self.class.name} - #{message}") + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 9323bfc7fb2..1d98d187805 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -56,7 +56,8 @@ module Gitlab name: project_path, path: project_path, repository_storage: storage_name, - namespace_id: group&.id + namespace_id: group&.id, + skip_disk_validation: true } project = Projects::CreateService.new(user, project_params).execute diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 28bbf3b384e..d1979bb7ed3 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,16 +149,21 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, + source_branch_sha: source_branch_sha, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, + target_branch_sha: target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb new file mode 100644 index 00000000000..72b75791bbb --- /dev/null +++ b/lib/gitlab/ci/ansi2html.rb @@ -0,0 +1,344 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Gitlab + module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + }.freeze + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 + }.freeze + + def self.convert(ansi, state = nil) + Converter.new.convert(ansi, state) + end + + class Converter + def on_0(s) reset() end + + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + + def on_31(s) set_fg_color(1) end + + def on_32(s) set_fg_color(2) end + + def on_33(s) set_fg_color(3) end + + def on_34(s) set_fg_color(4) end + + def on_35(s) set_fg_color(5) end + + def on_36(s) set_fg_color(6) end + + def on_37(s) set_fg_color(7) end + + def on_38(s) set_fg_color_256(s) end + + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + + def on_41(s) set_bg_color(1) end + + def on_42(s) set_bg_color(2) end + + def on_43(s) set_bg_color(3) end + + def on_44(s) set_bg_color(4) end + + def on_45(s) set_bg_color(5) end + + def on_46(s) set_bg_color(6) end + + def on_47(s) set_bg_color(7) end + + def on_48(s) set_bg_color_256(s) end + + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + + def on_91(s) set_fg_color(1, 'l') end + + def on_92(s) set_fg_color(2, 'l') end + + def on_93(s) set_fg_color(3, 'l') end + + def on_94(s) set_fg_color(4, 'l') end + + def on_95(s) set_fg_color(5, 'l') end + + def on_96(s) set_fg_color(6, 'l') end + + def on_97(s) set_fg_color(7, 'l') end + + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + + def on_101(s) set_bg_color(1, 'l') end + + def on_102(s) set_bg_color(2, 'l') end + + def on_103(s) set_bg_color(3, 'l') end + + def on_104(s) set_bg_color(4, 'l') end + + def on_105(s) set_bg_color(5, 'l') end + + def on_106(s) set_bg_color(6, 'l') end + + def on_107(s) set_bg_color(7, 'l') end + + def on_109(s) set_bg_color(9, 'l') end + + attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask + + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze + + def convert(stream, new_state) + reset_state + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset + + open_new_tag + + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) + handle_section(s) + elsif s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size + end + end + + close_open_tags() + + OpenStruct.new( + html: @out.force_encoding(Encoding.default_external), + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) + end + + def handle_section(s) + action = s[1] + timestamp = s[2] + section = s[3] + line = s.matched()[0...-5] # strips \r\033[0K + + @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' && terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + open_new_tag + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend + end + + evaluate_command_stack(stack) + end + + def open_new_tag + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + return if css_classes.empty? + + @out << %{<span class="#{css_classes.join(' ')}">} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{</span>} + @n_open_tags -= 1 + end + end + + def reset_state + @offset = 0 + @n_open_tags = 0 + @out = '' + reset + end + + def state + state = STATE_PARAMS.inject({}) do |h, param| + h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend + h + end + Base64.urlsafe_encode64(state.to_json) + end + + def restore_state(new_state, stream) + state = Base64.urlsafe_decode64(new_state) + state = JSON.parse(state, symbolize_names: true) + return if state[:offset].to_i > stream.size + + STATE_PARAMS.each do |param| + send("#{param}=".to_sym, state[param]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb new file mode 100644 index 00000000000..d10cc7802d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Build + module Policy + def self.fabricate(specs) + specifications = specs.to_h.map do |spec, value| + self.const_get(spec.to_s.camelize).new(value) + end + + specifications.compact + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb new file mode 100644 index 00000000000..b20d374288f --- /dev/null +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Build + module Policy + class Kubernetes < Policy::Specification + def initialize(spec) + unless spec.to_sym == :active + raise UnknownPolicyError + end + end + + def satisfied_by?(pipeline) + pipeline.has_kubernetes_active? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb new file mode 100644 index 00000000000..eadc0948d2f --- /dev/null +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -0,0 +1,43 @@ +module Gitlab + module Ci + module Build + module Policy + class Refs < Policy::Specification + def initialize(refs) + @patterns = Array(refs) + end + + def satisfied_by?(pipeline) + @patterns.any? do |pattern| + pattern, path = pattern.split('@', 2) + + matches_path?(path, pipeline) && + matches_pattern?(pattern, pipeline) + end + end + + private + + def matches_path?(path, pipeline) + return true unless path + + pipeline.project_full_path == path + end + + def matches_pattern?(pattern, pipeline) + return true if pipeline.tag? && pattern == 'tags' + return true if pipeline.branch? && pattern == 'branches' + return true if pipeline.source == pattern + return true if pipeline.source&.pluralize == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ pipeline.ref + else + pattern == pipeline.ref + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb new file mode 100644 index 00000000000..c317291f29d --- /dev/null +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Build + module Policy + ## + # Abstract class that defines an interface of job policy + # specification. + # + # Used for job's only/except policy configuration. + # + class Specification + UnknownPolicyError = Class.new(StandardError) + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb new file mode 100644 index 00000000000..7df7b542d91 --- /dev/null +++ b/lib/gitlab/ci/charts.rb @@ -0,0 +1,118 @@ +module Gitlab + module Ci + module Charts + module DailyInterval + def grouped_count(query) + query + .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + .transform_keys { |date| date.strftime(@format) } + end + + def interval_step + @interval_step ||= 1.day + end + end + + module MonthlyInterval + def grouped_count(query) + if Gitlab::Database.postgresql? + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) + else + query + .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") + .count(:created_at) + end + end + + def interval_step + @interval_step ||= 1.month + end + end + + class Chart + attr_reader :labels, :total, :success, :project, :pipeline_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @pipeline_times = [] + @project = project + + collect + end + + def collect + query = project.pipelines + .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + + totals_count = grouped_count(query) + success_count = grouped_count(query.success) + + current = @from + while current < @to + label = current.strftime(@format) + + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) + + current += interval_step + end + end + end + + class YearChart < Chart + include MonthlyInterval + + def initialize(*) + @to = Date.today.end_of_month + @from = @to.years_ago(1).beginning_of_month + @format = '%d %B %Y' + + super + end + end + + class MonthChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 30.days + @format = '%d %B' + + super + end + end + + class WeekChart < Chart + include DailyInterval + + def initialize(*) + @to = Date.today + @from = @to - 7.days + @format = '%d %B' + + super + end + end + + class PipelineTime < Chart + def collect + commits = project.pipelines.last(30) + + commits.each do |commit| + @labels << commit.short_sha + duration = commit.duration || 0 + @pipeline_times << (duration / 60) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb new file mode 100644 index 00000000000..0daddaa638c --- /dev/null +++ b/lib/gitlab/ci/mask_secret.rb @@ -0,0 +1,12 @@ +module Gitlab + module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end + end +end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb new file mode 100644 index 00000000000..3994a50772b --- /dev/null +++ b/lib/gitlab/ci/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb new file mode 100644 index 00000000000..8d82e1b288d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Base + attr_reader :pipeline, :project, :current_user + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + + @project = command.project + @current_user = command.current_user + end + + def perform! + raise NotImplementedError + end + + def break? + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb new file mode 100644 index 00000000000..d5e17a123df --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Create < Chain::Base + include Chain::Helpers + + def perform! + ::Ci::Pipeline.transaction do + pipeline.save! + + @command.seeds_block&.call(pipeline) + + ::Ci::CreatePipelineStagesService + .new(project, current_user) + .execute(pipeline) + end + rescue ActiveRecord::RecordInvalid => e + error("Failed to persist the pipeline: #{e}") + end + + def break? + !pipeline.persisted? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb new file mode 100644 index 00000000000..02d81286f21 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Helpers + def branch_exists? + return @is_branch if defined?(@is_branch) + + @is_branch = project.repository.branch_exists?(pipeline.ref) + end + + def tag_exists? + return @is_tag if defined?(@is_tag) + + @is_tag = project.repository.tag_exists?(pipeline.ref) + end + + def error(message) + pipeline.errors.add(:base, message) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb new file mode 100644 index 00000000000..015f2988327 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -0,0 +1,36 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Sequence + def initialize(pipeline, command, sequence) + @pipeline = pipeline + @completed = [] + + @sequence = sequence.map do |chain| + chain.new(pipeline, command) + end + end + + def build! + @sequence.each do |step| + step.perform! + + break if step.break? + + @completed << step + end + + @pipeline.tap do + yield @pipeline, self if block_given? + end + end + + def complete? + @completed.size == @sequence.size + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb new file mode 100644 index 00000000000..9a72de87bab --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Skip < Chain::Base + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + + def perform! + if skipped? + @pipeline.skip if @command.save_incompleted + end + end + + def skipped? + !@command.ignore_skip_ci && commit_message_skips_ci? + end + + def break? + skipped? + end + + private + + def commit_message_skips_ci? + return false unless @pipeline.git_commit_message + + @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb new file mode 100644 index 00000000000..4913a604079 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Abilities < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + + def perform! + unless project.builds_enabled? + return error('Pipelines are disabled!') + end + + unless allowed_to_trigger_pipeline? + if can?(current_user, :create_pipeline, project) + return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + else + return error('Insufficient permissions to create a new pipeline') + end + end + end + + def break? + @pipeline.errors.any? + end + + def allowed_to_trigger_pipeline? + if current_user + allowed_to_create? + else # legacy triggers don't have a corresponding user + !project.protected_for?(@pipeline.ref) + end + end + + def allowed_to_create? + return unless can?(current_user, :create_pipeline, project) + + access = Gitlab::UserAccess.new(current_user, project: project) + + if branch_exists? + access.can_update_branch?(@pipeline.ref) + elsif tag_exists? + access.can_create_tag?(@pipeline.ref) + else + true # Allow it for now and we'll reject when we check ref existence + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb new file mode 100644 index 00000000000..075504bcce5 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Config < Chain::Base + include Chain::Helpers + + def perform! + unless @pipeline.config_processor + unless @pipeline.ci_yaml_file + return error("Missing #{@pipeline.ci_yaml_file_path} file") + end + + if @command.save_incompleted && @pipeline.has_yaml_errors? + @pipeline.drop!(:config_error) + end + + return error(@pipeline.yaml_errors) + end + + unless @pipeline.has_stage_seeds? + return error('No stages / jobs for this pipeline.') + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb new file mode 100644 index 00000000000..70a4cfdbdea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Repository < Chain::Base + include Chain::Helpers + + def perform! + unless branch_exists? || tag_exists? + return error('Reference not found') + end + + ## TODO, we check commit in the service, that is why + # there is no repository access here. + # + unless pipeline.sha + return error('Commit not found') + end + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb new file mode 100644 index 00000000000..469fc094cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -0,0 +1,143 @@ +module Gitlab + module Ci + module Pipeline + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module Duration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb deleted file mode 100644 index 3208cc2bef6..00000000000 --- a/lib/gitlab/ci/pipeline_duration.rb +++ /dev/null @@ -1,141 +0,0 @@ -module Gitlab - module Ci - # # Introduction - total running time - # - # The problem this module is trying to solve is finding the total running - # time amongst all the jobs, excluding retries and pending (queue) time. - # We could reduce this problem down to finding the union of periods. - # - # So each job would be represented as a `Period`, which consists of - # `Period#first` as when the job started and `Period#last` as when the - # job was finished. A simple example here would be: - # - # * A (1, 3) - # * B (2, 4) - # * C (6, 7) - # - # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. - # C begins from 6, and ends to 7. Visually it could be viewed as: - # - # 0 1 2 3 4 5 6 7 - # AAAAAAA - # BBBBBBB - # CCCC - # - # The union of A, B, and C would be (1, 4) and (6, 7), therefore the - # total running time should be: - # - # (4 - 1) + (7 - 6) => 4 - # - # # The Algorithm - # - # The algorithm used here for union would be described as follow. - # First we make sure that all periods are sorted by `Period#first`. - # Then we try to merge periods by iterating through the first period - # to the last period. The goal would be merging all overlapped periods - # so that in the end all the periods are discrete. When all periods - # are discrete, we're free to just sum all the periods to get real - # running time. - # - # Here we begin from A, and compare it to B. We could find that - # before A ends, B already started. That is `B.first <= A.last` - # that is `2 <= 3` which means A and B are overlapping! - # - # When we found that two periods are overlapping, we would need to merge - # them into a new period and disregard the old periods. To make a new - # period, we take `A.first` as the new first because remember? we sorted - # them, so `A.first` must be smaller or equal to `B.first`. And we take - # `[A.last, B.last].max` as the new last because we want whoever ended - # later. This could be broken into two cases: - # - # 0 1 2 3 4 - # AAAAAAA - # BBBBBBB - # - # Or: - # - # 0 1 2 3 4 - # AAAAAAAAAA - # BBBB - # - # So that we need to take whoever ends later. Back to our example, - # after merging and discard A and B it could be visually viewed as: - # - # 0 1 2 3 4 5 6 7 - # DDDDDDDDDD - # CCCC - # - # Now we could go on and compare the newly created D and the old C. - # We could figure out that D and C are not overlapping by checking - # `C.first <= D.last` is `false`. Therefore we need to keep both C - # and D. The example would end here because there are no more jobs. - # - # After having the union of all periods, we just need to sum the length - # of all periods to get total time. - # - # (4 - 1) + (7 - 6) => 4 - # - # That is 4 is the answer in the example. - module PipelineDuration - extend self - - Period = Struct.new(:first, :last) do - def duration - last - first - end - end - - def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.builds.latest - .where(status: status).where.not(started_at: nil).order(:started_at) - - from_builds(builds) - end - - def from_builds(builds) - now = Time.now - - periods = builds.map do |b| - Period.new(b.started_at, b.finished_at || now) - end - - from_periods(periods) - end - - # periods should be sorted by `first` - def from_periods(periods) - process_duration(process_periods(periods)) - end - - private - - def process_periods(periods) - return periods if periods.empty? - - periods.drop(1).inject([periods.first]) do |result, current| - previous = result.last - - if overlap?(previous, current) - result[-1] = merge(previous, current) - result - else - result << current - end - end - end - - def overlap?(previous, current) - current.first <= previous.last - end - - def merge(previous, current) - Period.new(previous.first, [previous.last, current.last].max) - end - - def process_duration(periods) - periods.sum(&:duration) - end - end - end -end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb index e19aae35a81..bc97aa63b02 100644 --- a/lib/gitlab/ci/stage/seed.rb +++ b/lib/gitlab/ci/stage/seed.rb @@ -3,7 +3,9 @@ module Gitlab module Stage class Seed attr_reader :pipeline + delegate :project, to: :pipeline + delegate :size, to: :@jobs def initialize(pipeline, stage, jobs) @pipeline = pipeline diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 5b835bb669a..baf55b1fa07 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -27,6 +27,12 @@ module Gitlab end end + def extract_sections + read do |stream| + stream.extract_sections + end + end + def set(data) write do |stream| data = job.hide_secrets(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb new file mode 100644 index 00000000000..9bb0166c9e3 --- /dev/null +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -0,0 +1,97 @@ +module Gitlab + module Ci + class Trace + class SectionParser + def initialize(lines) + @lines = lines + end + + def parse! + @markers = {} + + @lines.each do |line, pos| + parse_line(line, pos) + end + end + + def sections + sanitize_markers.map do |name, markers| + start_, end_ = markers + + { + name: name, + byte_start: start_[:marker], + byte_end: end_[:marker], + date_start: start_[:timestamp], + date_end: end_[:timestamp] + } + end + end + + private + + def parse_line(line, line_start_position) + s = StringScanner.new(line) + until s.eos? + find_next_marker(s) do |scanner| + marker_begins_at = line_start_position + scanner.pointer + + if scanner.scan(Gitlab::Regex.build_trace_section_regex) + marker_ends_at = line_start_position + scanner.pointer + handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at) + true + else + false + end + end + end + end + + def sanitize_markers + @markers.select do |_, markers| + markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end + end + end + + def handle_line(action, time, name, marker_start, marker_end) + action = action.to_sym + timestamp = Time.at(time).utc + marker = if action == :start + marker_end + else + marker_start + end + + @markers[name] ||= [] + @markers[name] << { + name: name, + action: action, + timestamp: timestamp, + marker: marker + } + end + + def beginning_of_section_regex + @beginning_of_section_regex ||= /section_/.freeze + end + + def find_next_marker(s) + beginning_of_section_len = 8 + maybe_marker = s.exist?(beginning_of_section_regex) + + if maybe_marker.nil? + s.terminate + else + # repositioning at the beginning of the match + s.pos += maybe_marker - beginning_of_section_len + if block_given? + good_marker = yield(s) + # if not a good marker: Consuming the matched beginning_of_section_regex + s.pos += beginning_of_section_len unless good_marker + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 8503ecf8700..d52194f688b 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -56,13 +56,13 @@ module Gitlab end def html_with_state(state = nil) - ::Ci::Ansi2html.convert(stream, state) + ::Gitlab::Ci::Ansi2html.convert(stream, state) end def html(last_lines: nil) text = raw(last_lines: last_lines) buffer = StringIO.new(text) - ::Ci::Ansi2html.convert(buffer).html + ::Gitlab::Ci::Ansi2html.convert(buffer).html end def extract_coverage(regex) @@ -90,8 +90,25 @@ module Gitlab # so we just silently ignore error for now end + def extract_sections + return [] unless valid? + + lines = to_enum(:each_line_with_pos) + parser = SectionParser.new(lines) + + parser.parse! + parser.sections + end + private + def each_line_with_pos + stream.seek(0, IO::SEEK_SET) + stream.each_line do |line| + yield [line, stream.pos - line.bytesize] + end + end + def read_last_lines(limit) to_enum(:reverse_line).first(limit).reverse.join end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb new file mode 100644 index 00000000000..0bd78b03448 --- /dev/null +++ b/lib/gitlab/ci/yaml_processor.rb @@ -0,0 +1,189 @@ +module Gitlab + module Ci + class YamlProcessor + ValidationError = Class.new(StandardError) + + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + + attr_reader :cache, :stages, :jobs + + def initialize(config) + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + + initial_parsing + rescue Gitlab::Ci::Config::Loader::FormatError => e + raise ValidationError, e.message + end + + def builds + @jobs.map do |name, _| + build_attributes(name) + end + end + + def build_attributes(name) + job = @jobs[name.to_sym] || {} + + { stage_idx: @stages.index(job[:stage]), + stage: job[:stage], + commands: job[:commands], + tag_list: job[:tags] || [], + name: job[:name].to_s, + allow_failure: job[:ignore], + when: job[:when] || 'on_success', + environment: job[:environment_name], + coverage_regex: job[:coverage], + yaml_variables: yaml_variables(name), + options: { + image: job[:image], + services: job[:services], + artifacts: job[:artifacts], + cache: job[:cache], + dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], + after_script: job[:after_script], + environment: job[:environment], + retry: job[:retry] + }.compact } + end + + def pipeline_stage_builds(stage, pipeline) + selected_jobs = @jobs.select do |_, job| + next unless job[:stage] == stage + + only_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:only, {})) + except_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:except, {})) + + only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline) } + end + + selected_jobs.map { |_, job| build_attributes(job[:name]) } + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Gitlab::Ci::YamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + + private + + def initial_parsing + ## + # Global config + # + @before_script = @ci_config.before_script + @image = @ci_config.image + @after_script = @ci_config.after_script + @services = @ci_config.services + @variables = @ci_config.variables + @stages = @ci_config.stages + @cache = @ci_config.cache + + ## + # Jobs + # + @jobs = @ci_config.jobs + + @jobs.each do |name, job| + # logical validation for job + + validate_job_stage!(name, job) + validate_job_dependencies!(name, job) + validate_job_environment!(name, job) + end + end + + def yaml_variables(name) + variables = (@variables || {}) + .merge(job_variables(name)) + + variables.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def job_variables(name) + job = @jobs[name.to_sym] + return {} unless job + + job[:variables] || {} + end + + def validate_job_stage!(name, job) + return unless job[:stage] + + unless job[:stage].is_a?(String) && job[:stage].in?(@stages) + raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}" + end + end + + def validate_job_dependencies!(name, job) + return unless job[:dependencies] + + stage_index = @stages.index(job[:stage]) + + job[:dependencies].each do |dependency| + raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] + + unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" + end + end + end + + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 58f86abc5c4..7e7aaeeaa17 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,7 +1,7 @@ module Gitlab class ClosingIssueExtractor ISSUE_CLOSING_REGEX = begin - link_pattern = URI.regexp(%w(http https)) + link_pattern = Banzai::Filter::AutolinkFilter::LINK_PATTERN pattern = Gitlab.config.gitlab.issue_closing_pattern pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))") @@ -23,7 +23,8 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - @extractor.project.forked_from?(issue.project) # Don't extract issues on original project + # Don't extract issues from the project this project was forked from + @extractor.project.forked_from?(issue.project) end end end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index 84f9ecd3d23..e3678c914db 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -12,12 +12,7 @@ module Gitlab MissingEndDelimiter = Class.new(ParserError) def parse(text, our_path:, their_path:, parent_file: nil) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? + validate_text!(text) line_obj_index = 0 line_old = 1 @@ -32,15 +27,15 @@ module Gitlab full_line = line.delete("\n") if full_line == conflict_start - raise UnexpectedDelimiter unless type.nil? + validate_delimiter!(type.nil?) type = 'new' elsif full_line == conflict_middle - raise UnexpectedDelimiter unless type == 'new' + validate_delimiter!(type == 'new') type = 'old' elsif full_line == conflict_end - raise UnexpectedDelimiter unless type == 'old' + validate_delimiter!(type == 'old') type = nil elsif line[0] == '\\' @@ -59,6 +54,21 @@ module Gitlab lines end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end end end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 4ab5b3455a5..c169c8fe135 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -64,8 +64,11 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. - commit_attrs = commits_limited.map do |commit| - commit.hook_attrs(with_changed_files: true) + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 + commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do + commits_limited.map do |commit| + commit.hook_attrs(with_changed_files: true) + end end type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' @@ -83,7 +86,7 @@ module Gitlab user_name: user.name, user_username: user.username, user_email: user.email, - user_avatar: user.avatar_url, + user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index a6ec75da385..357f16936c6 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -29,6 +29,15 @@ module Gitlab adapter_name.casecmp('postgresql').zero? end + # Overridden in EE + def self.read_only? + false + end + + def self.read_write? + !self.read_only? + end + def self.version database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index fb14798efe6..2c35da8f1aa 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,6 +1,9 @@ module Gitlab module Database module MigrationHelpers + BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job + BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time + # Adds `created_at` and `updated_at` columns with timezone information. # # This method is an improved version of Rails' built-in method `add_timestamps`. @@ -653,6 +656,91 @@ into similar problems in the future (e.g. when new tables are created). EOF end end + + # Bulk queues background migration jobs for an entire table, batched by ID range. + # "Bulk" meaning many jobs will be pushed at a time for efficiency. + # If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`. + # + # model_class - The table being iterated over + # job_class_name - The background migration job class as a string + # batch_size - The maximum number of rows per job + # + # Example: + # + # class Route < ActiveRecord::Base + # include EachBatch + # self.table_name = 'routes' + # end + # + # bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes') + # + # Where the model_class includes EachBatch, and the background migration exists: + # + # class Gitlab::BackgroundMigration::ProcessRoutes + # def perform(start_id, end_id) + # # do something + # end + # end + def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) + raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + + jobs = [] + + model_class.each_batch(of: batch_size) do |relation| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + + if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE + # Note: This code path generally only helps with many millions of rows + # We push multiple jobs at a time to reduce the time spent in + # Sidekiq/Redis operations. We're using this buffer based approach so we + # don't need to run additional queries for every range. + BackgroundMigrationWorker.perform_bulk(jobs) + jobs.clear + end + + jobs << [job_class_name, [start_id, end_id]] + end + + BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + end + + # Queues background migration jobs for an entire table, batched by ID range. + # Each job is scheduled with a `delay_interval` in between. + # If you use a small interval, then some jobs may run at the same time. + # + # model_class - The table being iterated over + # job_class_name - The background migration job class as a string + # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) + # batch_size - The maximum number of rows per job + # + # Example: + # + # class Route < ActiveRecord::Base + # include EachBatch + # self.table_name = 'routes' + # end + # + # queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute) + # + # Where the model_class includes EachBatch, and the background migration exists: + # + # class Gitlab::BackgroundMigration::ProcessRoutes + # def perform(start_id, end_id) + # # do something + # end + # end + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) + raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + + model_class.each_batch(of: batch_size) do |relation, index| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + + # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for + # the same time, which is not helpful in most cases where we wish to + # spread the work over time. + BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id]) + end + end end end end diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb new file mode 100644 index 00000000000..4571ad122ce --- /dev/null +++ b/lib/gitlab/database/read_only_relation.rb @@ -0,0 +1,16 @@ +module Gitlab + module Database + # Module that can be injected into a ActiveRecord::Relation to make it + # read-only. + module ReadOnlyRelation + [:delete, :delete_all, :update, :update_all].each do |method| + define_method(method) do |*args| + raise( + ActiveRecord::ReadOnlyRecord, + "This relation is marked as read-only" + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 371cbe04b9b..c98eefbce25 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - base_sha == other.base_sha && - start_sha == other.start_sha && - head_sha == other.head_sha + shas_equal?(base_sha, other.base_sha) && + shas_equal?(start_sha, other.start_sha) && + shas_equal?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,6 +47,22 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end + + private + + def shas_equal?(sha1, sha2) + return true if sha1 == sha2 + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + length = [sha1.length, sha2.length].min + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + return false if length < Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 1dabd4ebdd0..599c3c5deab 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -5,7 +5,7 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false # Finding a viewer for a diff file happens based only on extension and whether the # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, @@ -27,16 +27,23 @@ module Gitlab @fallback_diff_refs = fallback_diff_refs end - def position(line) + def position(position_marker, position_type: :text) return unless diff_refs - Position.new( + data = { + diff_refs: diff_refs, + position_type: position_type.to_s, old_path: old_path, - new_path: new_path, - old_line: line.old_line, - new_line: line.new_line, - diff_refs: diff_refs - ) + new_path: new_path + } + + if position_type == :text + data.merge!(text_position_properties(position_marker)) + else + data.merge!(image_position_properties(position_marker)) + end + + Position.new(data) end def line_code(line) @@ -166,7 +173,7 @@ module Gitlab end def binary? - old_blob&.binary? || new_blob&.binary? + has_binary_notice? || old_blob&.binary? || new_blob&.binary? end def text? @@ -228,6 +235,14 @@ module Gitlab private + def text_position_properties(line) + { old_line: line.old_line, new_line: line.new_line } + end + + def image_position_properties(image_point) + image_point.to_h + end + def blobs_changed? old_blob && new_blob && old_blob.id != new_blob.id end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index a6007ebf531..88ae65cb468 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -22,7 +22,10 @@ module Gitlab end def diff_files - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445 + Gitlab::GitalyClient.allow_n_plus_1_calls do + @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } + end end def diff_file_with_old_path(old_path) diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb new file mode 100644 index 00000000000..5e923b9e602 --- /dev/null +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -0,0 +1,61 @@ +module Gitlab + module Diff + module Formatters + class BaseFormatter + attr_reader :old_path + attr_reader :new_path + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + attr_reader :position_type + + def initialize(attrs) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + + def key + [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || "")] + end + + def to_h + { + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha, + old_path: old_path, + new_path: new_path, + position_type: position_type + } + end + + def position_type + raise NotImplementedError + end + + def ==(other) + raise NotImplementedError + end + + def complete? + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb new file mode 100644 index 00000000000..ccd0d309972 --- /dev/null +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Diff + module Formatters + class ImageFormatter < BaseFormatter + attr_reader :width + attr_reader :height + attr_reader :x + attr_reader :y + + def initialize(attrs) + @x = attrs[:x] + @y = attrs[:y] + @width = attrs[:width] + @height = attrs[:height] + + super(attrs) + end + + def key + @key ||= super.push(x, y) + end + + def complete? + x && y && width && height + end + + def to_h + super.merge(width: width, height: height, x: x, y: y) + end + + def position_type + "image" + end + + def ==(other) + other.is_a?(self.class) && + x == other.x && + y == other.y + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb new file mode 100644 index 00000000000..01c7e9f51ab --- /dev/null +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -0,0 +1,49 @@ +module Gitlab + module Diff + module Formatters + class TextFormatter < BaseFormatter + attr_reader :old_line + attr_reader :new_line + + def initialize(attrs) + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + super(attrs) + end + + def key + @key ||= super.push(old_line, new_line) + end + + def complete? + old_line || new_line + end + + def to_h + super.merge(old_line: old_line, new_line: new_line) + end + + def line_age + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def position_type + "text" + end + + def ==(other) + other.is_a?(self.class) && + new_line == other.new_line && + old_line == other.old_line + end + end + end + end +end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb new file mode 100644 index 00000000000..65332dfd239 --- /dev/null +++ b/lib/gitlab/diff/image_point.rb @@ -0,0 +1,23 @@ +module Gitlab + module Diff + class ImagePoint + attr_reader :width, :height, :x, :y + + def initialize(width, height, x, y) + @width = width + @height = height + @x = x + @y = y + end + + def to_h + { + width: width, + height: height, + x: x, + y: y + } + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 919965100ae..010b4be7b40 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -2,9 +2,10 @@ module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + mark = super(line_inline_diffs) do |text, left:, right:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end + mark.html_safe end private diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f80afb20f0c..bd0a9502a5e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,37 +1,25 @@ -# Defines a specific location, identified by paths and line numbers, +# Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab module Diff class Position - attr_reader :old_path - attr_reader :new_path - attr_reader :old_line - attr_reader :new_line - attr_reader :base_sha - attr_reader :start_sha - attr_reader :head_sha - + attr_accessor :formatter + + delegate :old_path, + :new_path, + :base_sha, + :start_sha, + :head_sha, + :old_line, + :new_line, + :position_type, to: :formatter + + # A position can belong to a text line or to an image coordinate + # it depends of the position_type argument. + # Text position will have: new_line and old_line + # Image position will have: width, height, x, y def initialize(attrs = {}) - if diff_file = attrs[:diff_file] - attrs[:diff_refs] = diff_file.diff_refs - attrs[:old_path] = diff_file.old_path - attrs[:new_path] = diff_file.new_path - end - - if diff_refs = attrs[:diff_refs] - attrs[:base_sha] = diff_refs.base_sha - attrs[:start_sha] = diff_refs.start_sha - attrs[:head_sha] = diff_refs.head_sha - end - - @old_path = attrs[:old_path] - @new_path = attrs[:new_path] - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - - @old_line = attrs[:old_line] - @new_line = attrs[:new_line] + @formatter = get_formatter_class(attrs[:position_type]).new(attrs) end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -46,27 +34,23 @@ module Gitlab end def encode_with(coder) - coder['attributes'] = self.to_h + coder['attributes'] = formatter.to_h end def key - @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + formatter.key end def ==(other) - other.is_a?(self.class) && key == other.key + other.is_a?(self.class) && + other.diff_refs == diff_refs && + other.old_path == old_path && + other.new_path == new_path && + other.formatter == formatter end def to_h - { - old_path: old_path, - new_path: new_path, - old_line: old_line, - new_line: new_line, - base_sha: base_sha, - start_sha: start_sha, - head_sha: head_sha - } + formatter.to_h end def inspect @@ -74,23 +58,15 @@ module Gitlab end def complete? - file_path.present? && - (old_line || new_line) && - diff_refs.complete? + file_path.present? && formatter.complete? && diff_refs.complete? end def to_json(opts = nil) - JSON.generate(self.to_h, opts) + JSON.generate(formatter.to_h, opts) end def type - if old_line && new_line - nil - elsif new_line - 'new' - else - 'old' - end + formatter.line_age end def unchanged? @@ -149,6 +125,17 @@ module Gitlab diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end + + def get_formatter_class(type) + type ||= "text" + + case type + when 'image' + Gitlab::Diff::Formatters::ImageFormatter + else + Gitlab::Diff::Formatters::TextFormatter + end + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index abd401224d8..c4c60d1dfee 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,7 +2,7 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck - CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze + DEFAULT_CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze @@ -20,7 +20,7 @@ module Gitlab attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found attr_reader :failed_files - def initialize(branch:, ce_repo: CE_REPO) + def initialize(branch:, ce_repo: DEFAULT_CE_REPO) @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @@ -132,7 +132,7 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) - step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{ce_repo} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", # Don't use --check here because it can result in a 0-exit status even @@ -284,13 +284,18 @@ module Gitlab EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch was found in the EE repository. + If you're a community contributor, don't worry, someone from + GitLab Inc. will take care of this, and you don't have to do anything. + If you're willing to help, and are ok to contribute to EE as well, + you're welcome to help. You could follow the instructions below. + #{conflicting_files_msg} We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch that includes changes from `#{ce_branch}` but also specific changes than can be applied cleanly to EE/master. In some cases, the conflicts are trivial and you can ignore the warning from this job. As always, - use your best judgment! + use your best judgement! There are different ways to create such branch: diff --git a/lib/gitlab/gcp/model.rb b/lib/gitlab/gcp/model.rb new file mode 100644 index 00000000000..195391f0e3c --- /dev/null +++ b/lib/gitlab/gcp/model.rb @@ -0,0 +1,13 @@ +module Gitlab + module Gcp + module Model + def table_name_prefix + "gcp_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end + end +end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index b984492d369..455814a9159 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -29,6 +29,8 @@ module Gitlab # http://gitlab.com/some/link/#1234, and code `puts #1234`' # class ReferenceRewriter + RewriteError = Class.new(StandardError) + def initialize(text, source_project, current_user) @text = text @source_project = source_project @@ -61,6 +63,10 @@ module Gitlab cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference + if cross_reference.nil? + raise RewriteError, "Unspecified reference detected for #{referable.class.name}" + end + new_text = before + cross_reference + after substitution_valid?(new_text) ? cross_reference : reference end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8c9acbc9fbe..c78fe63f9b5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) @@ -57,6 +57,15 @@ module Gitlab def version Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) end + + def check_namespace!(*objects) + expected_namespace = self.name + '::' + objects.each do |object| + unless object.class.name.start_with?(expected_namespace) + raise ArgumentError, "expected object in #{expected_namespace}, got #{object}" + end + end + end end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 8d96826f6ee..a4336facee5 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -32,6 +32,8 @@ module Gitlab else blob = repository.lookup(sha) + next unless blob.is_a?(Rugged::Blob) + new( id: blob.oid, size: blob.size, diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 5ee6669050c..1957c254c28 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -352,7 +352,7 @@ module Gitlab end def stats - Gitlab::Git::CommitStats.new(self) + Gitlab::Git::CommitStats.new(@repository, self) end def to_patch(options = {}) @@ -413,6 +413,10 @@ module Gitlab end end + def merge_commit? + parent_ids.size > 1 + end + private def init_from_hash(hash) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 00acb4763e9..6bf49a0af18 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -10,12 +10,29 @@ module Gitlab # Instantiate a CommitStats object # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 - def initialize(commit) + def initialize(repo, commit) @id = commit.id @additions = 0 @deletions = 0 @total = 0 + repo.gitaly_migrate(:commit_stats) do |is_enabled| + if is_enabled + gitaly_stats(repo, commit) + else + rugged_stats(commit) + end + end + end + + def gitaly_stats(repo, commit) + stats = repo.gitaly_commit_client.commit_stats(@id) + @additions = stats.additions + @deletions = stats.deletions + @total = @additions + @deletions + end + + def rugged_stats(commit) diff = commit.rugged_diff_from_parent diff.each_patch do |p| diff --git a/lib/gitlab/git/committer.rb b/lib/gitlab/git/committer.rb deleted file mode 100644 index 1f4bcf7a3a0..00000000000 --- a/lib/gitlab/git/committer.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Gitlab - module Git - class Committer - attr_reader :name, :email, :gl_id - - def self.from_user(user) - new(user.name, user.email, Gitlab::GlId.gl_id(user)) - end - - def initialize(name, email, gl_id) - @name = name - @email = email - @gl_id = gl_id - end - - def ==(other) - [name, email, gl_id] == [other.name, other.email, other.gl_id] - end - end - end -end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a23c8cf0dd1..ca94b4baa59 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -24,41 +24,13 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze - class << self - # The maximum size of a diff to display. - def size_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit - else - find_size_limit - end - end - - # The maximum size before a diff is collapsed. - def collapse_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit - else - find_collapse_limit - end - end - - def find_size_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 200.kilobytes - else - 100.kilobytes - end - end + # The maximum size of a diff to display. + SIZE_LIMIT = 100.kilobytes - def find_collapse_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 100.kilobytes - else - 10.kilobytes - end - end + # The maximum size before a diff is collapsed. + COLLAPSE_LIMIT = 10.kilobytes + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -172,7 +144,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.size_limit + @too_large = @diff.bytesize >= SIZE_LIMIT else @too_large end @@ -190,7 +162,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end def collapse! @@ -206,6 +178,10 @@ module Gitlab Diff.binary_message(@old_path, @new_path) end + def has_binary_notice? + @diff.start_with?('Binary') + end + private def init_from_rugged(rugged) @@ -271,14 +247,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= self.class.size_limit + if size >= SIZE_LIMIT too_large! return true end end end - if !expanded && size >= self.class.collapse_limit + if !expanded && size >= COLLAPSE_LIMIT collapse! return true end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index cc35d77c6e4..e29a1f7afa1 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -22,22 +22,22 @@ module Gitlab File.exist?(path) end - def trigger(gl_id, oldrev, newrev, ref) + def trigger(gl_id, gl_username, oldrev, newrev, ref) return [true, nil] unless exists? Bundler.with_clean_env do case name when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) + call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + call_update_hook(gl_id, gl_username, oldrev, newrev, ref) end end end private - def call_receive_hook(gl_id, oldrev, newrev, ref) + def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") exit_status = false @@ -45,6 +45,7 @@ module Gitlab vars = { 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, 'GL_REPOSITORY' => repository.gl_repository @@ -80,16 +81,21 @@ module Gitlab [exit_status, exit_message] end - def call_update_hook(gl_id, oldrev, newrev, ref) + def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) Dir.chdir(repo_path) do - stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) - [status.success?, stderr.presence || stdout] + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username + } + stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end end def retrieve_error_message(stderr, stdout) - err_message = stderr.gets - err_message.blank? ? stdout.gets : err_message + err_message = stderr.read + err_message = err_message.blank? ? stdout.read : err_message + err_message.gsub(/\R/, "<br>").html_safe end end end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index ea8a87a1290..c327e9b1616 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -5,12 +5,13 @@ module Gitlab attr_accessor :oldrev, :newrev, :ref - def execute(committer, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = committer.gl_id - @oldrev = oldrev - @newrev = newrev - @ref = ref + def execute(pusher, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = pusher.gl_id + @gl_username = pusher.name + @oldrev = oldrev + @newrev = newrev + @ref = ref %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) @@ -29,7 +30,7 @@ module Gitlab def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, oldrev, newrev, ref) + hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 9e6fca8c80c..d835dcca8ba 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,17 +1,24 @@ module Gitlab module Git class OperationService - attr_reader :committer, :repository + include Gitlab::Git::Popen - def initialize(committer, new_repository) - committer = Gitlab::Git::Committer.from_user(committer) if committer.is_a?(User) - @committer = committer + WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do + alias_method :repo_created?, :repo_created + alias_method :branch_created?, :branch_created + end - # Refactoring aid - unless new_repository.is_a?(Gitlab::Git::Repository) - raise "expected a Gitlab::Git::Repository, got #{new_repository}" + attr_reader :user, :repository + + def initialize(user, new_repository) + if user + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + @user = user end + # Refactoring aid + Gitlab::Git.check_namespace!(new_repository) + @repository = new_repository end @@ -105,7 +112,7 @@ module Gitlab ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name update_ref_in_hooks(ref, newrev, oldrev) - [newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)] + WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) end def find_oldrev_from_branch(newrev, branch) @@ -128,7 +135,7 @@ module Gitlab def with_hooks(ref, newrev, oldrev) Gitlab::Git::HooksService.new.execute( - committer, + user, repository, oldrev, newrev, @@ -145,13 +152,15 @@ module Gitlab # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - _, status = Gitlab::Popen.popen( + + output, status = popen( command, repository.path) do |stdin| stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") end unless status.zero? + Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") raise Gitlab::Git::CommitError.new( "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ " Please refresh and try again.") diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 25fa62ce4bd..3d2fc471d28 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -5,17 +5,21 @@ require 'open3' module Gitlab module Git module Popen - def popen(cmd, path) + def popen(cmd, path, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end - vars = { "PWD" => path } + path ||= Dir.pwd + vars['PWD'] = path options = { chdir: path } @cmd_output = "" @cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + yield(stdin) if block_given? + stdin.close + @cmd_output << stdout.read @cmd_output << stderr.read @cmd_status = wait_thr.value.exitstatus diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index efa13590a2c..0f059bef808 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -19,13 +19,12 @@ module Gitlab InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) + CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the storage path, not the storage name - def create(storage_path, name, bare: true, symlink_hooks_to: nil) - repo_path = File.join(storage_path, name) - repo_path += '.git' unless repo_path.end_with?('.git') - + # Unlike `new`, `create` takes the repository path + def create(repo_path, bare: true, symlink_hooks_to: nil) FileUtils.mkdir_p(repo_path, mode: 0770) # Equivalent to `git --git-path=#{repo_path} init [--bare]` @@ -54,14 +53,15 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver - # 'path' must be the path to a _bare_ git repository, e.g. - # /path/to/my-repo.git + # This initializer method is only used on the client side (gitlab-ce). + # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -72,8 +72,6 @@ module Gitlab delegate :empty?, to: :rugged - delegate :exists?, to: :gitaly_repository_client - def ==(other) path == other.path end @@ -101,6 +99,18 @@ module Gitlab @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end + def exists? + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + gitaly_repository_client.exists? + else + circuit_breaker.perform do + File.exist?(File.join(@path, 'refs')) + end + end + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -180,6 +190,28 @@ module Gitlab end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -385,7 +417,13 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(self, c) } + gitaly_migrate(:find_commits) do |is_enabled| + if is_enabled + gitaly_commit_client.find_commits(options) + else + raw_log(options).map { |c| Commit.decorate(self, c) } + end + end end # Used in gitaly-ruby @@ -474,7 +512,15 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + iterator = gitaly_migrate(:diff_between) do |is_enabled| + if is_enabled + gitaly_commit_client.diff(from, to, options.merge(paths: paths)) + else + diff_patches(from, to, options, *paths) + end + end + + Gitlab::Git::DiffCollection.new(iterator, options) end # Returns a RefName for a given SHA @@ -489,7 +535,7 @@ module Gitlab # Not found -> ["", 0] # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - Gitlab::Popen.popen(args, @path).first.split.last + popen(args, @path).first.split.last end end end @@ -610,47 +656,167 @@ module Gitlab # TODO: implement this method end - def add_branch(branch_name, committer:, target:) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object + def add_branch(branch_name, user:, target:) + gitaly_migrate(:operation_user_create_branch) do |is_enabled| + if is_enabled + gitaly_add_branch(branch_name, user, target) + else + rugged_add_branch(branch_name, user, target) + end + end + end - OperationService.new(committer, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + def add_tag(tag_name, user:, target:, message: nil) + gitaly_migrate(:operation_user_add_tag) do |is_enabled| + if is_enabled + gitaly_add_tag(tag_name, user: user, target: target, message: message) + else + rugged_add_tag(tag_name, user: user, target: target, message: message) + end + end end - def add_tag(tag_name, committer:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object + def rm_branch(branch_name, user:) + gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_delete_branch(branch_name, user) + else + OperationService.new(user, self).rm_branch(find_branch(branch_name)) + end + end + end - committer = Committer.from_user(committer) if committer.is_a?(User) + def rm_tag(tag_name, user:) + gitaly_migrate(:operation_user_delete_tag) do |is_enabled| + if is_enabled + gitaly_operations_client.rm_tag(tag_name, user) + else + Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) + end + end + end + + def find_tag(name) + tags.find { |tag| tag.name == name } + end + + def merge(user, source_sha, target_branch, message) + committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) + + OperationService.new(user, self).with_branch(target_branch) do |start_commit| + our_commit = start_commit.sha + their_commit = source_sha + + raise 'Invalid merge target' unless our_commit + raise 'Invalid merge source' unless their_commit + + merge_index = rugged.merge_commits(our_commit, their_commit) + break if merge_index.conflicts? - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message options = { + parents: [our_commit, their_commit], + tree: merge_index.write_tree(rugged), message: message, - tagger: Gitlab::Git.committer_hash(email: committer.email, name: committer.name) + author: committer, + committer: committer } + + commit_id = create_commit(options) + + yield commit_id + + commit_id end + rescue Gitlab::Git::CommitError # when merge_index.conflicts? + nil + end - OperationService.new(committer, self).add_tag(tag_name, target_object.oid, options) + def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + Gitlab::Git.check_namespace!(commit, start_repository) + + revert_tree_id = check_revert_content(commit, start_commit.sha) + raise CreateTreeError unless revert_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) + end + end + + def check_revert_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << { mainline: 1 } if target_commit.merge_commit? + + revert_index = rugged.revert_commit(*args) + return false if revert_index.conflicts? + + tree_id = revert_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + + def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) + raise CreateTreeError unless cherry_pick_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) + end end - def rm_branch(branch_name, committer:) - OperationService.new(committer, self).rm_branch(find_branch(branch_name)) + def check_cherry_pick_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << 1 if target_commit.merge_commit? + + cherry_pick_index = rugged.cherrypick_commit(*args) + return false if cherry_pick_index.conflicts? + + tree_id = cherry_pick_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id end - def rm_tag(tag_name, committer:) - OperationService.new(committer, self).rm_tag(find_tag(tag_name)) + def diff_exists?(sha1, sha2) + rugged.diff(sha1, sha2).size > 0 end - def find_tag(name) - tags.find { |tag| tag.name == name } + def user_to_committer(user) + Gitlab::Git.committer_hash(email: user.email, name: user.name) + end + + def create_commit(params = {}) + params[:message].delete!("\r") + + Rugged::Commit.create(rugged, params) end # Delete the specified branch from the repository @@ -672,9 +838,7 @@ module Gitlab end command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - message, status = Gitlab::Popen.popen( - command, - path) do |stdin| + message, status = popen(command, path) do |stdin| stdin.write(instructions.join) end @@ -798,14 +962,18 @@ module Gitlab end def with_repo_branch_commit(start_repository, start_branch_name) - raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository) + Gitlab::Git.check_namespace!(start_repository) return yield nil if start_repository.empty_repo? if start_repository == self yield commit(start_branch_name) else - sha = start_repository.commit(start_branch_name).sha + start_commit = start_repository.commit(start_branch_name) + + return yield nil unless start_commit + + sha = start_commit.sha if branch_commit = commit(sha) yield branch_commit @@ -820,9 +988,9 @@ module Gitlab def with_repo_tmp_commit(start_repository, start_branch_name, sha) tmp_ref = fetch_ref( - start_repository.path, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - "refs/tmp/#{SecureRandom.hex}/head" + start_repository, + source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + target_ref: "refs/tmp/#{SecureRandom.hex}" ) yield commit(sha) @@ -834,8 +1002,9 @@ module Gitlab with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) + true else - raise Rugged::ReferenceError, 'source repository is empty' + false end end end @@ -853,13 +1022,27 @@ module Gitlab end end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) + def write_ref(ref_path, ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + + command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] + input = "update #{ref_path}\x00#{ref}\x00\x00" + output, status = circuit_breaker.perform do + popen(command, path) { |stdin| stdin.write(input) } + end + + raise GitError, output unless status.zero? end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - message, status = run_git(args) + def fetch_ref(source_repository, source_ref:, target_ref:) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| + if is_enabled + gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) + else + local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) + end + end # Make sure ref was created, and raise Rugged::ReferenceError when not raise Rugged::ReferenceError, message if status != 0 @@ -868,9 +1051,9 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args) + def run_git(args, env: {}) circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) + popen([Gitlab.config.git.bin_path, *args], path, env) end end @@ -894,11 +1077,17 @@ module Gitlab # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + end + + def gitaly_operations_client + @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) end def gitaly_ref_client @@ -913,26 +1102,35 @@ module Gitlab @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) end - def gitaly_migrate(method, &block) - Gitlab::GitalyClient.migrate(method, &block) + def gitaly_operation_client + @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) + end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e raise NoRepository.new(e) rescue GRPC::BadStatus => e raise CommandError.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) end private # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 + branches = Gitlab::GitalyClient.allow_n_plus_1_calls do + rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + end sort_branches(branches, sort_by) end @@ -1232,6 +1430,33 @@ module Gitlab false end + def gitaly_add_tag(tag_name, user:, target:, message: nil) + gitaly_operations_client.add_tag(tag_name, user, target, message) + end + + def rugged_add_tag(tag_name, user:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) + } + end + + Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + rescue Rugged::TagError + raise TagExistsError + end + def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1274,6 +1499,46 @@ module Gitlab file.write(gitattributes_content) end end + + def gitaly_add_branch(branch_name, user, target) + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex + end + + def rugged_add_branch(branch_name, user, target) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(user, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def local_fetch_ref(source_path, source_ref:, target_ref:) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + run_git(args) + end + + def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_resolver.address(source_repository.storage) + gitaly_token = gitaly_resolver.token(source_repository.storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + + run_git(args, env: env) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 2b5785a1f08..92a6a672534 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -3,6 +3,8 @@ module Gitlab module Git class RevList + include Gitlab::Git::Popen + attr_reader :oldrev, :newrev, :path_to_repo def initialize(path_to_repo:, newrev:, oldrev: nil) @@ -26,10 +28,10 @@ module Gitlab private def execute(args) - output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" end output.split("\n") diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index e28be4b8a38..08e6c29abad 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -11,6 +11,7 @@ module Gitlab end CircuitOpen = Class.new(Inaccessible) + Misconfiguration = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 9ea9367d4b7..1eaa2d83fb6 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -28,14 +28,26 @@ module Gitlab def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| - hash[storage_name] = new(storage_name) + hash[storage_name] = build(storage_name) end end cached_circuitbreakers[storage] end - def initialize(storage, hostname = Gitlab::Environment.hostname) + def self.build(storage, hostname = Gitlab::Environment.hostname) + config = Gitlab.config.repositories.storages[storage] + + if !config.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) + elsif !config['path'].present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) + else + new(storage, hostname) + end + end + + def initialize(storage, hostname) @storage = storage @hostname = hostname @@ -64,6 +76,10 @@ module Gitlab recent_failure || too_many_failures end + def failure_info + @failure_info ||= get_failure_info + end + # Memoizing the `storage_available` call means we only do it once per # request when the storage is available. # @@ -121,10 +137,12 @@ module Gitlab end end - def failure_info - @failure_info ||= get_failure_info + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end + private + def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -134,10 +152,6 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end end end end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 2d723147f4f..1564e94b7f7 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -78,7 +78,7 @@ module Gitlab def failing_circuit_breakers @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.new(storage_name, hostname) + CircuitBreaker.build(storage_name, hostname) end end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb new file mode 100644 index 00000000000..297c043d054 --- /dev/null +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -0,0 +1,47 @@ +module Gitlab + module Git + module Storage + class NullCircuitBreaker + # These will have actual values + attr_reader :storage, + :hostname + + # These will always have nil values + attr_reader :storage_path, + :failure_wait_time, + :failure_reset_time, + :storage_timeout + + def initialize(storage, hostname, error: nil) + @storage = storage + @hostname = hostname + @error = error + end + + def perform + @error ? raise(@error) : yield + end + + def circuit_broken? + !!@error + end + + def failure_count_threshold + 1 + end + + def last_failure + circuit_broken? ? Time.now : nil + end + + def failure_count + circuit_broken? ? 1 : 0 + end + + def failure_info + Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + end + end + end + end +end diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb new file mode 100644 index 00000000000..da74719ae87 --- /dev/null +++ b/lib/gitlab/git/user.rb @@ -0,0 +1,27 @@ +module Gitlab + module Git + class User + attr_reader :username, :name, :email, :gl_id + + def self.from_gitlab(gitlab_user) + new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + end + + # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628 + def self.from_gitaly(gitaly_user) + new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + end + + def initialize(username, name, email, gl_id) + @username = username + @name = name + @email = email + @gl_id = gl_id + end + + def ==(other) + [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] + end + end + end +end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb new file mode 100644 index 00000000000..d651c931a38 --- /dev/null +++ b/lib/gitlab/git/wiki.rb @@ -0,0 +1,115 @@ +module Gitlab + module Git + class Wiki + DuplicatePageError = Class.new(StandardError) + + CommitDetails = Struct.new(:name, :email, :message) do + def to_h + { name: name, email: email, message: message } + end + end + + def self.default_ref + 'master' + end + + # Initialize with a Gitlab::Git::Repository instance + def initialize(repository) + @repository = repository + end + + def repository_exists? + @repository.exists? + end + + def write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def delete_page(page_path, commit_details) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) + nil + end + + def update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + + def pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + + def page(title:, version: nil, dir: nil) + if version + version = Gitlab::Git::Commit.find(@repository, version).id + end + + gollum_page = gollum_wiki.page(title, version, dir) + return unless gollum_page + + new_page(gollum_page) + end + + def file(name, version) + version ||= self.class.default_ref + gollum_file = gollum_wiki.file(name, version) + return unless gollum_file + + Gitlab::Git::WikiFile.new(gollum_file) + end + + def page_versions(page_path) + current_page = gollum_page_by_path(page_path) + current_page.versions.map do |gollum_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) + new_version(gollum_page, gollum_git_commit.id) + end + end + + def preview_slug(title, format) + gollum_wiki.preview_page(title, '', format).url_path + end + + private + + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + + def gollum_page_by_path(page_path) + page_name = Gollum::Page.canonicalize_filename(page_path) + page_dir = File.split(page_path).first + + gollum_wiki.paged(page_name, page_dir) + end + + def new_page(gollum_page) + Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) + end + + def new_version(gollum_page, commit_id) + commit = Gitlab::Git::Commit.find(@repository, commit_id) + Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + end + + def assert_type!(object, klass) + unless object.is_a?(klass) + raise ArgumentError, "expected a #{klass}, got #{object.inspect}" + end + end + end + end +end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb new file mode 100644 index 00000000000..527f2a44dea --- /dev/null +++ b/lib/gitlab/git/wiki_file.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiFile + attr_reader :mime_type, :raw_data, :name + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::File is not serializable we must get all the data from + # 'gollum_file' during initialization, and NOT store it in an instance + # variable. + def initialize(gollum_file) + @mime_type = gollum_file.mime_type + @raw_data = gollum_file.raw_data + @name = gollum_file.name + end + end + end +end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb new file mode 100644 index 00000000000..a06bac4414f --- /dev/null +++ b/lib/gitlab/git/wiki_page.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + class WikiPage + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::Page is not serializable we must get all the data from + # 'gollum_page' during initialization, and NOT store it in an instance + # variable. + # + # Note that 'version' is a WikiPageVersion instance which it itself + # serializable. That means it's OK to store 'version' in an instance + # variable. + def initialize(gollum_page, version) + @url_path = gollum_page.url_path + @title = gollum_page.title + @format = gollum_page.format + @path = gollum_page.path + @raw_data = gollum_page.raw_data + @name = gollum_page.name + @historical = gollum_page.historical? + + @version = version + end + + def historical? + @historical + end + + def text_data + return @text_data if defined?(@text_data) + + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end + end + end +end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb new file mode 100644 index 00000000000..55f1afedcab --- /dev/null +++ b/lib/gitlab/git/wiki_page_version.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiPageVersion + attr_reader :commit, :format + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are + # serializable. + def initialize(commit, format) + @commit = commit + @format = format + end + + delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 62d1ecae676..42b59c106e2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -16,7 +16,9 @@ module Gitlab account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', - receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', + read_only: 'The repository is temporarily read-only. Please try again later.', + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -159,6 +161,14 @@ module Gitlab end def check_push_access!(changes) + if project.repository_read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + end + if deploy_key check_deploy_key_push_access! elsif user diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1fe5155c093..98f1f45b338 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,7 @@ module Gitlab class GitAccessWiki < GitAccess ERROR_MESSAGES = { + read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze @@ -17,6 +18,10 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + if Gitlab::Database.read_only? + raise UnauthorizedError, ERROR_MESSAGES[:read_only] + end + true end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index a3c6b21a6a1..2e3e4fc3f1f 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -11,7 +11,7 @@ module Gitlab return false if ref_name.start_with?('refs/remotes/') Gitlab::Utils.system_silent( - %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) + %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name})) end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a3dc2cd0b60..cf36106e23d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -10,7 +10,25 @@ module Gitlab OPT_OUT = 3 end + class TooManyInvocationsError < StandardError + attr_reader :call_site, :invocation_count, :max_call_stack + + def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack) + @call_site = call_site + @invocation_count = invocation_count + @max_call_stack = max_call_stack + stacks = most_invoked_stack.join('\n') if most_invoked_stack + + msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?" + msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks + + super(msg) + end + end + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze + MAXIMUM_GITALY_CALLS = 30 + CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new private_constant :MUTEX @@ -53,6 +71,8 @@ module Gitlab # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. def self.call(storage, service, rpc, request) + enforce_gitaly_request_limits(:call) + metadata = request_metadata(storage) metadata = yield(metadata) if block_given? stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend @@ -60,7 +80,16 @@ module Gitlab def self.request_metadata(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) - { metadata: { 'authorization' => "Bearer #{encoded_token}" } } + metadata = { + 'authorization' => "Bearer #{encoded_token}", + 'client_name' => CLIENT_NAME + } + + feature_stack = Thread.current[:gitaly_feature_stack] + feature = feature_stack && feature_stack[0] + metadata['call_site'] = feature.to_s if feature + + { metadata: metadata } end def self.token(storage) @@ -107,12 +136,107 @@ module Gitlab private_class_method :opt_into_all_features? def self.migrate(feature, status: MigrationStatus::OPT_IN) + # Enforce limits at both the `migrate` and `call` sites to ensure that + # problems are not hidden by a feature being disabled + enforce_gitaly_request_limits(:migrate) + is_enabled = feature_enabled?(feature, status: status) metric_name = feature.to_s metric_name += "_gitaly" if is_enabled Gitlab::Metrics.measure(metric_name) do - yield is_enabled + # Some migrate calls wrap other migrate calls + allow_n_plus_1_calls do + feature_stack = Thread.current[:gitaly_feature_stack] ||= [] + feature_stack.unshift(feature) + begin + yield is_enabled + ensure + feature_stack.shift + Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? + end + end + end + end + + # Ensures that Gitaly is not being abuse through n+1 misuse etc + def self.enforce_gitaly_request_limits(call_site) + # Only count limits in request-response environments (not sidekiq for example) + return unless RequestStore.active? + + # This is this actual number of times this call was made. Used for information purposes only + actual_call_count = increment_call_count("gitaly_#{call_site}_actual") + + # Do no enforce limits in production + return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"] + + # Check if this call is nested within a allow_n_plus_1_calls + # block and skip check if it is + return if get_call_count(:gitaly_call_count_exception_block_depth) > 0 + + # This is the count of calls outside of a `allow_n_plus_1_calls` block + # It is used for enforcement but not statistics + permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted") + + count_stack + + return if permitted_call_count <= MAXIMUM_GITALY_CALLS + + raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks) + end + + def self.allow_n_plus_1_calls + return yield unless RequestStore.active? + + begin + increment_call_count(:gitaly_call_count_exception_block_depth) + yield + ensure + decrement_call_count(:gitaly_call_count_exception_block_depth) + end + end + + def self.get_call_count(key) + RequestStore.store[key] || 0 + end + private_class_method :get_call_count + + def self.increment_call_count(key) + RequestStore.store[key] ||= 0 + RequestStore.store[key] += 1 + end + private_class_method :increment_call_count + + def self.decrement_call_count(key) + RequestStore.store[key] -= 1 + end + private_class_method :decrement_call_count + + # Returns an estimate of the number of Gitaly calls made for this + # request + def self.get_request_count + return 0 unless RequestStore.active? + + gitaly_migrate_count = get_call_count("gitaly_migrate_actual") + gitaly_call_count = get_call_count("gitaly_call_actual") + + # Using the maximum of migrate and call_count will provide an + # indicator of how many Gitaly calls will be made, even + # before a feature is enabled. This provides us with a single + # metric, but not an exact number, but this tradeoff is acceptable + if gitaly_migrate_count > gitaly_call_count + gitaly_migrate_count + else + gitaly_call_count + end + end + + def self.reset_counts + return unless RequestStore.active? + + %w[migrate call].each do |call_site| + RequestStore.store["gitaly_#{call_site}_actual"] = 0 + RequestStore.store["gitaly_#{call_site}_permitted"] = 0 end end @@ -121,8 +245,56 @@ module Gitlab path.read.chomp end + def self.timestamp(t) + Google::Protobuf::Timestamp.new(seconds: t.to_i) + end + def self.encode(s) + return "" if s.nil? + s.dup.force_encoding(Encoding::ASCII_8BIT) end + + def self.encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) + end + + # Count a stack. Used for n+1 detection + def self.count_stack + return unless RequestStore.active? + + stack_string = caller.drop(1).join("\n") + + RequestStore.store[:stack_counter] ||= Hash.new + + count = RequestStore.store[:stack_counter][stack_string] || 0 + RequestStore.store[:stack_counter][stack_string] = count + 1 + end + private_class_method :count_stack + + # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection + def self.max_call_count + return 0 unless RequestStore.active? + + stack_counter = RequestStore.store[:stack_counter] + return 0 unless stack_counter + + stack_counter.values.max + end + private_class_method :max_call_count + + # Returns the stacks that calls Gitaly the most times. Used for n+1 detection + def self.max_stacks + return nil unless RequestStore.active? + + stack_counter = RequestStore.store[:stack_counter] + return nil unless stack_counter + + max = max_call_count + return nil if max.zero? + + stack_counter.select { |_, v| v == max }.keys + end + private_class_method :max_stacks end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0825a3a7694..a2b50f2507e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -32,20 +32,38 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value end + def diff(from, to, options = {}) + from_id = case from + when NilClass + EMPTY_TREE_ID + when Rugged::Commit + from.oid + else + from + end + + to_id = case to + when NilClass + EMPTY_TREE_ID + when Rugged::Commit + to.oid + else + to + end + + request_params = diff_between_commits_request_params(from_id, to_id, options) + + call_commit_diff(request_params, options) + end + def diff_from_parent(commit, options = {}) - request_params = commit_diff_request_params(commit, options) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - request_params[:enforce_limits] = options.fetch(:limits, true) - request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params = diff_from_parent_request_params(commit, options) - request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) - GitalyClient::DiffStitcher.new(response) + call_commit_diff(request_params, options) end def commit_deltas(commit) - request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) + request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) response.flat_map { |msg| msg.deltas } @@ -204,16 +222,59 @@ module Gitlab response.sum(&:data) end + def commit_stats(revision) + request = Gitaly::CommitStatsRequest.new( + repository: @gitaly_repo, + revision: GitalyClient.encode(revision) + ) + GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) + end + + def find_commits(options) + request = Gitaly::FindCommitsRequest.new( + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + disable_walk: options[:disable_walk] + ) + request.after = GitalyClient.timestamp(options[:after]) if options[:after] + request.before = GitalyClient.timestamp(options[:before]) if options[:before] + request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + + request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + + consume_commits_response(response) + end + private - def commit_diff_request_params(commit, options = {}) + def call_commit_diff(request_params, options = {}) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + request_params[:enforce_limits] = options.fetch(:limits, true) + request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + + request = Gitaly::CommitDiffRequest.new(request_params) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + GitalyClient::DiffStitcher.new(response) + end + + def diff_from_parent_request_params(commit, options = {}) parent_id = commit.parent_ids.first || EMPTY_TREE_ID + diff_between_commits_request_params(parent_id, commit.id, options) + end + + def diff_between_commits_request_params(from_id, to_id, options) { repository: @gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - paths: options.fetch(:paths, []) + left_commit_id: from_id, + right_commit_id: to_id, + paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } } end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb new file mode 100644 index 00000000000..bd7c345ac01 --- /dev/null +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -0,0 +1,39 @@ +module Gitlab + module GitalyClient + class NamespaceService + def initialize(storage) + @storage = storage + end + + def exists?(name) + request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:namespace_exists, request).exists + end + + def add(name) + request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:add_namespace, request) + end + + def remove(name) + request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) + + gitaly_client_call(:remove_namespace, request) + end + + def rename(from, to) + request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) + + gitaly_client_call(:rename_namespace, request) + end + + private + + def gitaly_client_call(type, request) + GitalyClient.call(@storage, :namespace_service, type, request) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb new file mode 100644 index 00000000000..81ddaf13e10 --- /dev/null +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -0,0 +1,79 @@ +module Gitlab + module GitalyClient + class OperationService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def rm_tag(tag_name, user) + request = Gitaly::UserDeleteTagRequest.new( + repository: @gitaly_repo, + tag_name: GitalyClient.encode(tag_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def add_tag(tag_name, user, target, message) + request = Gitaly::UserCreateTagRequest.new( + repository: @gitaly_repo, + user: Util.gitaly_user(user), + tag_name: GitalyClient.encode(tag_name), + target_revision: GitalyClient.encode(target), + message: GitalyClient.encode(message.to_s) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + elsif response.exists + raise Gitlab::Git::Repository::TagExistsError + end + + Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Repository::InvalidRef, e + end + + def user_create_branch(branch_name, user, start_point) + request = Gitaly::UserCreateBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user), + start_point: GitalyClient.encode(start_point) + ) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_create_branch, request) + if response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + end + + branch = response.branch + return nil unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + end + + def user_delete_branch(branch_name, user) + request = Gitaly::UserDeleteBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8ef873d5848..8214b7d63fa 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -57,6 +57,14 @@ module Gitlab branch_names.count end + # TODO implement a more efficient RPC for this https://gitlab.com/gitlab-org/gitaly/issues/616 + def has_local_branches? + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request).first + + response&.names.present? + end + def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by @@ -155,19 +163,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map do |gitaly_tag| - if gitaly_tag.target_commit.present? - gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - @repository, - encode!(gitaly_tag.name.dup), - gitaly_tag.id, - gitaly_commit, - encode!(gitaly_tag.message.chomp) - ) - end + message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 177a1284f38..fdf912214e0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -53,6 +53,11 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request) end + + def create_repository + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :create_repository, request) + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 8fc937496af..da43c616b94 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -2,14 +2,39 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path) + def repository(repository_storage, relative_path, gl_repository) Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, + gl_repository: gl_repository, git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) ) end + + def gitaly_user(gitlab_user) + return unless gitlab_user + + Gitaly::User.new( + gl_id: Gitlab::GlId.gl_id(gitlab_user), + name: GitalyClient.encode(gitlab_user.name), + email: GitalyClient.encode(gitlab_user.email) + ) + end + + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) + if gitaly_tag.target_commit.present? + commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) + end + + Gitlab::Git::Tag.new( + repository, + Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), + gitaly_tag.id, + commit, + Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) + ) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9bcc579278f..3a666c2268b 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,6 +20,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') + gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 025f826e65f..413872d7e08 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -34,6 +34,21 @@ module Gitlab end end + def subkeys_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + raw_keys = GPGME::Key.find(:public, fingerprints) + + raw_keys.each_with_object({}) do |raw_key, grouped_subkeys| + primary_subkey_id = raw_key.primary_subkey.keyid + + grouped_subkeys[primary_subkey_id] = raw_key.subkeys[1..-1].map do |s| + { keyid: s.keyid, fingerprint: s.fingerprint } + end + end + end + end + def user_infos_from_key(key) using_tmp_keychain do fingerprints = CurrentKeyChain.fingerprints_from_key(key) @@ -69,11 +84,17 @@ module Gitlab def optimistic_using_tmp_keychain previous_dir = current_home_dir - Dir.mktmpdir do |dir| - GPGME::Engine.home_dir = dir - yield - end + tmp_dir = Dir.mktmpdir + GPGME::Engine.home_dir = tmp_dir + yield ensure + # Ignore any errors when removing the tmp directory, as we may run into a + # race condition: + # The `gpg-agent` agent process may clean up some files as well while + # `FileUtils.remove_entry` is iterating the directory and removing all + # its contained files and directories recursively, which could raise an + # error. + FileUtils.remove_entry(tmp_dir, true) GPGME::Engine.home_dir = previous_dir end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 86bd9f5b125..0f4ba6f83fc 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -43,7 +43,9 @@ module Gitlab # key belonging to the keyid. # This way we can add the key to the temporary keychain and extract # the proper signature. - gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + # NOTE: the invoked method is #fingerprint but it's only returning + # 16 characters (the format used by keyid) instead of 40. + gpg_key = find_gpg_key(verified_signature.fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) @@ -74,7 +76,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -98,6 +100,10 @@ module Gitlab def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + + def find_gpg_key(keyid) + GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) + end end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index e085eab26c9..1991911ef6a 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -9,8 +9,8 @@ module Gitlab GpgSignature .select(:id, :commit_sha, :project_id) .where('gpg_key_id IS NULL OR verification_status <> ?', GpgSignature.verification_statuses[:verified]) - .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) - .find_each { |sig| sig.gpg_commit.update_signature!(sig) } + .where(gpg_key_primary_keyid: @gpg_key.keyids) + .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end end end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 5a31e56cb30..635f52131f9 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -22,7 +22,7 @@ module Gitlab def base_and_ancestors return ancestors_base unless Group.supports_nested_groups? - base_and_ancestors_cte.apply_to(model.all) + read_only(base_and_ancestors_cte.apply_to(model.all)) end # Returns a relation that includes the descendants_base set of groups @@ -30,7 +30,7 @@ module Gitlab def base_and_descendants return descendants_base unless Group.supports_nested_groups? - base_and_descendants_cte.apply_to(model.all) + read_only(base_and_descendants_cte.apply_to(model.all)) end # Returns a relation that includes the base groups, their ancestors, @@ -67,11 +67,13 @@ module Gitlab union = SQL::Union.new([model.unscoped.from(ancestors_table), model.unscoped.from(descendants_table)]) - model + relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) .from("(#{union.to_sql}) #{model.table_name}") + + read_only(relation) end private @@ -107,5 +109,12 @@ module Gitlab def groups_table model.arel_table end + + def read_only(relation) + # relations using a CTE are not safe to use with update_all as it will + # throw away the CTE, hence we mark them as read-only. + relation.extend(Gitlab::Database::ReadOnlyRelation) + relation + end end end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index eef97f54962..afaa59b1018 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -58,11 +58,11 @@ module Gitlab end def repository_storages - @repository_storage ||= Gitlab::CurrentSettings.current_application_settings.repository_storages + storages_paths.keys end def storages_paths - @storage_paths ||= Gitlab.config.repositories.storages + Gitlab.config.repositories.storages end def exec_with_timeout(cmd_args, *args, &block) @@ -125,7 +125,7 @@ module Gitlab end def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" } + Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } rescue Gitlab::Git::Storage::Inaccessible nil end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5d106b5c075..bdc0f04b56b 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -17,7 +17,8 @@ module Gitlab 'it' => 'Italiano', 'uk' => 'Українська', 'ja' => '日本語', - 'ko' => '한국어' + 'ko' => '한국어', + 'nl_NL' => 'Nederlands' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2171c6c7bbb..dec8b4c5acd 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -53,6 +53,7 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules + - :cluster - :services - :hooks - protected_branches: diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 380b336395d..a76cf1addc0 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,6 +8,8 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + cluster: 'Gcp::Cluster', + clusters: 'Gcp::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index cdbdfa10d0e..da43bd0af4b 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -113,7 +113,7 @@ module Gitlab def kubeconfig_embed_ca_pem(config, ca_pem) cluster = config.dig(:clusters, 0, :cluster) - cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + cluster[:'certificate-authority-data'] = Base64.strict_encode64(ca_pem) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index fb68627dedf..e60ceba27c8 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,7 +16,7 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true else diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index cd7e4ca7b7e..0afaa2306b5 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -22,8 +22,8 @@ module Gitlab Gitlab::LDAP::Config.new(provider) end - def users(field, value, limit = nil) - options = user_options(field, value, limit) + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -72,20 +72,24 @@ module Gitlab private - def user_options(field, value, limit) - options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } + def user_options(fields, value, limit) + options = { + attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + base: config.base + } + options[:size] = limit if limit - if field.to_sym == :dn + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + options[:base] = value options[:scope] = Net::LDAP::SearchScope_BaseObject - options[:filter] = user_filter else - options[:base] = config.base - options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) end - options + options.merge(filter: user_filter(filter)) end def user_filter(filter = nil) diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 4fbc5fa5262..3123da17fd9 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -3,6 +3,10 @@ module Gitlab module LDAP class AuthHash < Gitlab::OAuth::AuthHash + def uid + Gitlab::LDAP::Person.normalize_dn(super) + end + private def get_info(key) diff --git a/lib/gitlab/ldap/dn.rb b/lib/gitlab/ldap/dn.rb new file mode 100644 index 00000000000..d6142dc6549 --- /dev/null +++ b/lib/gitlab/ldap/dn.rb @@ -0,0 +1,301 @@ +# -*- ruby encoding: utf-8 -*- + +# Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` +# +# For our purposes, this class is used to normalize DNs in order to allow proper +# comparison. +# +# E.g. DNs should be compared case-insensitively (in basically all LDAP +# implementations or setups), therefore we downcase every DN. + +## +# Objects of this class represent an LDAP DN ("Distinguished Name"). A DN +# ("Distinguished Name") is a unique identifier for an entry within an LDAP +# directory. It is made up of a number of other attributes strung together, +# to identify the entry in the tree. +# +# Each attribute that makes up a DN needs to have its value escaped so that +# the DN is valid. This class helps take care of that. +# +# A fully escaped DN needs to be unescaped when analysing its contents. This +# class also helps take care of that. +module Gitlab + module LDAP + class DN + FormatError = Class.new(StandardError) + MalformedError = Class.new(FormatError) + UnsupportedError = Class.new(FormatError) + + def self.normalize_value(given_value) + dummy_dn = "placeholder=#{given_value}" + normalized_dn = new(*dummy_dn).to_normalized_s + normalized_dn.sub(/\Aplaceholder=/, '') + end + + ## + # Initialize a DN, escaping as required. Pass in attributes in name/value + # pairs. If there is a left over argument, it will be appended to the dn + # without escaping (useful for a base string). + # + # Most uses of this class will be to escape a DN, rather than to parse it, + # so storing the dn as an escaped String and parsing parts as required + # with a state machine seems sensible. + def initialize(*args) + if args.length > 1 + initialize_array(args) + else + initialize_string(args[0]) + end + end + + ## + # Parse a DN into key value pairs using ASN from + # http://tools.ietf.org/html/rfc2253 section 3. + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def each_pair + state = :key + key = StringIO.new + value = StringIO.new + hex_buffer = "" + + @dn.each_char.with_index do |char, dn_index| + case state + when :key then + case char + when 'a'..'z', 'A'..'Z' then + state = :key_normal + key << char + when '0'..'9' then + state = :key_oid + key << char + when ' ' then state = :key + else raise(MalformedError, "Unrecognized first character of an RDN attribute type name \"#{char}\"") + end + when :key_normal then + case char + when '=' then state = :value + when 'a'..'z', 'A'..'Z', '0'..'9', '-', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN attribute type name character \"#{char}\"") + end + when :key_oid then + case char + when '=' then state = :value + when '0'..'9', '.', ' ' then key << char + else raise(MalformedError, "Unrecognized RDN OID attribute type name character \"#{char}\"") + end + when :value then + case char + when '\\' then state = :value_normal_escape + when '"' then state = :value_quoted + when ' ' then state = :value + when '#' then + state = :value_hexstring + value << char + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else + state = :value_normal + value << char + end + when :value_normal then + case char + when '\\' then state = :value_normal_escape + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + when '+' then raise(UnsupportedError, "Multivalued RDNs are not supported") + else value << char + end + when :value_normal_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal_escape_hex + hex_buffer = char + else + state = :value_normal + value << char + end + when :value_normal_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_normal + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Invalid escaped hex code \"\\#{hex_buffer}#{char}\"") + end + when :value_quoted then + case char + when '\\' then state = :value_quoted_escape + when '"' then state = :value_end + else value << char + end + when :value_quoted_escape then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted_escape_hex + hex_buffer = char + else + state = :value_quoted + value << char + end + when :value_quoted_escape_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_quoted + value << "#{hex_buffer}#{char}".to_i(16).chr + else raise(MalformedError, "Expected the second character of a hex pair inside a double quoted value, but got \"#{char}\"") + end + when :value_hexstring then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring_hex + value << char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the first character of a hex pair, but got \"#{char}\"") + end + when :value_hexstring_hex then + case char + when '0'..'9', 'a'..'f', 'A'..'F' then + state = :value_hexstring + value << char + else raise(MalformedError, "Expected the second character of a hex pair, but got \"#{char}\"") + end + when :value_end then + case char + when ' ' then state = :value_end + when ',' then + state = :key + yield key.string.strip, rstrip_except_escaped(value.string, dn_index) + key = StringIO.new + value = StringIO.new + else raise(MalformedError, "Expected the end of an attribute value, but got \"#{char}\"") + end + else raise "Fell out of state machine" + end + end + + # Last pair + raise(MalformedError, 'DN string ended unexpectedly') unless + [:value, :value_normal, :value_hexstring, :value_end].include? state + + yield key.string.strip, rstrip_except_escaped(value.string, @dn.length) + end + + def rstrip_except_escaped(str, dn_index) + str_ends_with_whitespace = str.match(/\s\z/) + + if str_ends_with_whitespace + dn_part_ends_with_escaped_whitespace = @dn[0, dn_index].match(/\\(\s+)\z/) + + if dn_part_ends_with_escaped_whitespace + dn_part_rwhitespace = dn_part_ends_with_escaped_whitespace[1] + num_chars_to_remove = dn_part_rwhitespace.length - 1 + str = str[0, str.length - num_chars_to_remove] + else + str.rstrip! + end + end + + str + end + + ## + # Returns the DN as an array in the form expected by the constructor. + def to_a + a = [] + self.each_pair { |key, value| a << key << value } unless @dn.empty? + a + end + + ## + # Return the DN as an escaped string. + def to_s + @dn + end + + ## + # Return the DN as an escaped and normalized string. + def to_normalized_s + self.class.new(*to_a).to_s.downcase + end + + # https://tools.ietf.org/html/rfc4514 section 2.4 lists these exceptions + # for DN values. All of the following must be escaped in any normal string + # using a single backslash ('\') as escape. The space character is left + # out here because in a "normalized" string, spaces should only be escaped + # if necessary (i.e. leading or trailing space). + NORMAL_ESCAPES = [',', '+', '"', '\\', '<', '>', ';', '='].freeze + + # The following must be represented as escaped hex + HEX_ESCAPES = { + "\n" => '\0a', + "\r" => '\0d' + }.freeze + + # Compiled character class regexp using the keys from the above hash, and + # checking for a space or # at the start, or space at the end, of the + # string. + ESCAPE_RE = Regexp.new("(^ |^#| $|[" + + NORMAL_ESCAPES.map { |e| Regexp.escape(e) }.join + + "])") + + HEX_ESCAPE_RE = Regexp.new("([" + + HEX_ESCAPES.keys.map { |e| Regexp.escape(e) }.join + + "])") + + ## + # Escape a string for use in a DN value + def self.escape(string) + escaped = string.gsub(ESCAPE_RE) { |char| "\\" + char } + escaped.gsub(HEX_ESCAPE_RE) { |char| HEX_ESCAPES[char] } + end + + private + + def initialize_array(args) + buffer = StringIO.new + + args.each_with_index do |arg, index| + if index.even? # key + buffer << "," if index > 0 + buffer << arg + else # value + buffer << "=" + buffer << self.class.escape(arg) + end + end + + @dn = buffer.string + end + + def initialize_string(arg) + @dn = arg.to_s + end + + ## + # Proxy all other requests to the string object, because a DN is mainly + # used within the library as a string + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(method, *args, &block) + @dn.send(method, *args, &block) + end + + ## + # Redefined to be consistent with redefined `method_missing` behavior + def respond_to?(sym, include_private = false) + @dn.respond_to?(sym, include_private) + end + end + end +end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 4d6f8ac79de..38d7a9ba2f5 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -17,6 +17,12 @@ module Gitlab adapter.user('dn', dn) end + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + def self.disabled_via_active_directory?(dn, adapter) adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end @@ -30,6 +36,26 @@ module Gitlab ] end + def self.normalize_dn(dn) + ::Gitlab::LDAP::DN.new(dn).to_normalized_s + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original DN \"#{dn}\" due to error during normalization attempt: #{e.message}") + + dn + end + + # Returns the UID in a normalized form. + # + # 1. Excess spaces are stripped + # 2. The string is downcased (for case-insensitivity) + def self.normalize_uid(uid) + ::Gitlab::LDAP::DN.normalize_value(uid) + rescue ::Gitlab::LDAP::DN::FormatError => e + Rails.logger.info("Returning original UID \"#{uid}\" due to error during normalization attempt: #{e.message}") + + uid + end + def initialize(entry, provider) Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @@ -52,7 +78,9 @@ module Gitlab attribute_value(:email) end - delegate :dn, to: :entry + def dn + self.class.normalize_dn(entry.dn) + end private diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3bf27b37ae6..1793097363e 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -17,41 +17,19 @@ module Gitlab end end - def initialize(auth_hash) - super - update_user_attributes - end - def save super('LDAP') end # instance methods - def gl_user - @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user + def find_user + find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) - end - - def update_user_attributes - if persisted? - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) - - # For a new identity set extern_uid to the LDAP DN - # For an existing identity with matching email but changed DN, update the DN. - # For an existing identity with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid - end - end - def changed? gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 9f432673a6e..344784c866f 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -4,6 +4,15 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom + DEFAULT_CONFIG = { + enabled: false, + port: 143, + ssl: false, + start_tls: false, + mailbox: 'inbox', + idle_timeout: 60 + }.freeze + class << self def enabled? config[:enabled] && config[:address] @@ -22,16 +31,10 @@ module Gitlab def fetch_config return {} unless File.exist?(config_file) - rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' - all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys - - config = all_config[:incoming_email] || {} - config[:enabled] = false if config[:enabled].nil? - config[:port] = 143 if config[:port].nil? - config[:ssl] = false if config[:ssl].nil? - config[:start_tls] = false if config[:start_tls].nil? - config[:mailbox] = 'inbox' if config[:mailbox].nil? - config[:idle_timeout] = 60 if config[:idle_timeout].nil? + config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys[:incoming_email] || {} + config = DEFAULT_CONFIG.merge(config) do |_key, oldval, newval| + newval.nil? ? oldval : newval + end if config[:enabled] && config[:address] gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) @@ -45,6 +48,10 @@ module Gitlab config end + def rails_env + @rails_env ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development' + end + def config_file ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb deleted file mode 100644 index 306923902e0..00000000000 --- a/lib/gitlab/markdown/pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Gitlab - module Markdown - class Pipeline - def self.[](name) - name ||= :full - const_get("#{name.to_s.camelize}Pipeline") - end - - def self.filters - [] - end - - def self.transform_context(context) - context - end - - def self.html_pipeline - @html_pipeline ||= HTML::Pipeline.new(filters) - end - - class << self - %i(call to_document to_html).each do |meth| - define_method(meth) do |text, context| - context = transform_context(context) - - html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 6023fa1820f..f42168c720e 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -3,6 +3,10 @@ module Gitlab module Middleware class Go + include ActionView::Helpers::TagHelper + + PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze + def initialize(app) @app = app end @@ -10,17 +14,20 @@ module Gitlab def call(env) request = Rack::Request.new(env) - if go_request?(request) - render_go_doc(request) - else - @app.call(env) - end + render_go_doc(request) || @app.call(env) end private def render_go_doc(request) - body = go_body(request) + return unless go_request?(request) + + path = project_path(request) + return unless path + + body = go_body(path) + return unless body + response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' }) response.finish end @@ -29,11 +36,13 @@ module Gitlab request["go-get"].to_i == 1 && request.env["PATH_INFO"].present? end - def go_body(request) - project_url = URI.join(Gitlab.config.gitlab.url, project_path(request)) + def go_body(path) + project_url = URI.join(Gitlab.config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n" + meta_tag = tag :meta, name: 'go-import', content: "#{import_prefix} git #{project_url}.git" + head_tag = content_tag :head, meta_tag + content_tag :html, head_tag end def strip_url(url) @@ -44,6 +53,10 @@ module Gitlab path_info = request.env["PATH_INFO"] path_info.sub!(/^\//, '') + project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) + return unless project_path_match + path = project_path_match[1] + # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. # In a traditional project with a single namespace, this would denote repo # `namespace/project` with subpath `path1/path2/../pathN`, but with nested @@ -51,7 +64,7 @@ module Gitlab # `path2/../pathN`, for example. # We find all potential project paths out of the path segments - path_segments = path_info.split('/') + path_segments = path.split('/') simple_project_path = path_segments.first(2).join('/') # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb new file mode 100644 index 00000000000..0de0cddcce4 --- /dev/null +++ b/lib/gitlab/middleware/read_only.rb @@ -0,0 +1,88 @@ +module Gitlab + module Middleware + class ReadOnly + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + API_VERSIONS = (3..4) + + def initialize(app) + @app = app + @whitelisted = internal_routes + end + + def call(env) + @env = env + + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + error_message = 'You cannot do writing operations on a read-only GitLab instance' + + if json_request? + return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] + else + rack_flash.alert = error_message + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(env) + end + + private + + def internal_routes + API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } + end + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def logout_route + route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + request.path.end_with?('.git/git-upload-pack') + end + + def lfs_route + request.path.end_with?('/info/lfs/objects/batch') + end + end + end +end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 1f331b1e91d..5b5ed449f94 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -13,7 +13,7 @@ module Gitlab end def provider - @provider ||= Gitlab::Utils.force_utf8(auth_hash.provider.to_s) + @provider ||= auth_hash.provider.to_s end def name diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7704bf715e4..47c2a422387 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -13,6 +13,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash update_profile if sync_profile_from_provider? + add_or_update_user_identities end def persisted? @@ -32,7 +33,7 @@ module Gitlab block_after_save = needs_blocking? - Users::UpdateService.new(gl_user).execute! + Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block if block_after_save @@ -44,47 +45,56 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider + return @gl_user if defined?(@gl_user) - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end + @gl_user = find_user + end - if signup_enabled? - @user ||= build_new_user - end + def find_user + user = find_by_uid_and_provider - if external_provider? && @user - @user.external = true - end + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user - @user + user end protected - def find_or_create_ldap_user + def add_or_update_user_identities + return unless gl_user + + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user return unless ldap_person - # If a corresponding person exists with same uid in a LDAP server, - # check if the user already has a GitLab account. user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) if user - # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) - else - log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." - user = find_by_uid_and_provider - if user.nil? - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - user = build_new_user - end - log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." - user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) + return user end - user + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) end def auto_link_ldap_user? @@ -108,9 +118,9 @@ module Gitlab end def find_ldap_person(auth_hash, adapter) - by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) end def ldap_config @@ -152,7 +162,7 @@ module Gitlab end def build_new_user - user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) + user_params = user_attributes.merge(skip_confirmation: true) Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb new file mode 100644 index 00000000000..981ef8faa9a --- /dev/null +++ b/lib/gitlab/pages.rb @@ -0,0 +1,5 @@ +module Gitlab + module Pages + VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7c02c9c5c48..e68160c8faf 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -33,6 +33,7 @@ module Gitlab explore favicon.ico files + google_api groups health_check help diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 58f6245579a..bd677ec4bf3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -65,5 +65,9 @@ module Gitlab "can contain only lowercase letters, digits, and '-'. " \ "Must start with a letter, and cannot end with '-'" end + + def build_trace_section_regex + @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze + end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 0f323a9e8b2..e0a9d1dee77 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -10,41 +10,20 @@ module Gitlab super('SAML') end - def gl_user - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end - - @user ||= find_by_uid_and_provider - - if auto_link_saml_user? - @user ||= find_by_email - end + def find_user + user = find_by_uid_and_provider - if signup_enabled? - @user ||= build_new_user - end + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? - if external_users_enabled? && @user + if external_users_enabled? && user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - @user.external = - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - false - else - true - end + user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? end - @user - end - - def find_by_email - if auth_hash.has_attribute?(:email) - user = ::User.find_by(email: auth_hash.email.downcase) - user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user - user - end + user end def changed? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 81ecdf43ef9..a37112ae5c4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -65,7 +65,7 @@ module Gitlab # Init new repository # - # storage - project's storage path + # storage - project's storage name # name - project path with namespace # # Ex. @@ -73,7 +73,19 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + relative_path = name.dup + relative_path << '.git' unless relative_path.end_with?('.git') + + gitaly_migrate(:create_repository) do |is_enabled| + if is_enabled + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + repository.gitaly_repository_client.create_repository + true + else + repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + end + end rescue => err Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false @@ -210,10 +222,18 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) - path = full_path(storage, name) - FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).add(name) + else + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + end + end rescue Errno::EEXIST => e Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Remove directory from repositories storage @@ -224,7 +244,15 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) - FileUtils.rm_r(full_path(storage, name), force: true) + Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).remove(name) + else + FileUtils.rm_r(full_path(storage, name), force: true) + end + end + rescue GRPC::InvalidArgument => e + raise ArgumentError, e.message end # Move namespace directory inside repositories storage @@ -234,9 +262,17 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) - return false if exists?(storage, new_name) || !exists?(storage, old_name) + Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + if enabled + gitaly_namespace_client(storage).rename(old_name, new_name) + else + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) + end + end + rescue GRPC::InvalidArgument + false end def url_to_repo(path) @@ -260,7 +296,13 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - File.exist?(full_path(storage, dir_name)) + Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + if enabled + gitaly_namespace_client(storage).exists?(dir_name) + else + File.exist?(full_path(storage, dir_name)) + end + end end protected @@ -337,6 +379,14 @@ module Gitlab Bundler.with_original_env { Popen.popen(cmd, nil, vars) } end + def gitaly_namespace_client(storage_path) + storage, _value = Gitlab.config.repositories.storages.find do |storage, value| + value['path'] == storage_path + end + + Gitlab::GitalyClient::NamespaceService.new(storage) + end + def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound, GRPC::BadStatus => e diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 104280f520a..d7d24eeb37b 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,7 +25,7 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']} "\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 222021e8802..f30c771837a 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -12,8 +12,9 @@ module Gitlab # # Project.where("id IN (#{sql})") class Union - def initialize(relations) + def initialize(relations, remove_duplicates: true) @relations = relations + @remove_duplicates = remove_duplicates end def to_sql @@ -25,7 +26,11 @@ module Gitlab @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end - fragments.join("\nUNION\n") + fragments.join("\n#{union_keyword}\n") + end + + def union_keyword + @remove_duplicates ? 'UNION' : 'UNION ALL' end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 703adae12cb..1caa791c1be 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,9 @@ module Gitlab class UrlSanitizer + ALLOWED_SCHEMES = %w[http https ssh git].freeze + def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) + regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -11,21 +13,20 @@ module Gitlab def self.valid?(url) return false unless url.present? - Addressable::URI.parse(url.strip) + uri = Addressable::URI.parse(url.strip) - true + ALLOWED_SCHEMES.include?(uri.scheme) rescue Addressable::URI::InvalidURIError false end def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url.to_s.strip) - %i[user password].each do |symbol| credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) end @credentials = credentials + @url = parse_url(url) end def sanitized_url @@ -49,12 +50,30 @@ module Gitlab private + def parse_url(url) + url = url.to_s.strip + match = url.match(%r{\A(?:git|ssh|http(?:s?))\://(?:(.+)(?:@))?(.+)}) + raw_credentials = match[1] if match + + if raw_credentials.present? + url.sub!("#{raw_credentials}@", '') + + user, password = raw_credentials.split(':') + @credentials ||= { user: user.presence, password: password.presence } + end + + url = Addressable::URI.parse(url) + url.password = password if password.present? + url.user = user if user.present? + url + end + def generate_full_url return @url unless valid_credentials? @full_url = @url.dup - @full_url.password = credentials[:password] - @full_url.user = credentials[:user] + @full_url.password = credentials[:password] if credentials[:password].present? + @full_url.user = credentials[:user] if credentials[:user].present? @full_url end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 3cf26625108..3f3ba77d47f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,12 +9,28 @@ module Gitlab def uncached_data license_usage_data.merge(system_usage_data) + .merge(features_usage_data) + .merge(components_usage_data) end def to_json(force_refresh: false) data(force_refresh: force_refresh).to_json end + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + def system_usage_data { counts: { @@ -22,12 +38,17 @@ module Gitlab ci_builds: ::Ci::Build.count, ci_internal_pipelines: ::Ci::Pipeline.internal.count, ci_external_pipelines: ::Ci::Pipeline.external.count, + ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, + ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, ci_pipeline_schedules: ::Ci::PipelineSchedule.count, + auto_devops_enabled: ::ProjectAutoDevops.enabled.count, + auto_devops_disabled: ::ProjectAutoDevops.disabled.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, + gcp_clusters: ::Gcp::Cluster.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, @@ -50,18 +71,28 @@ module Gitlab } end - def license_usage_data - usage_data = { - uuid: current_application_settings.uuid, - hostname: Gitlab.config.gitlab.host, - version: Gitlab::VERSION, - active_user_count: User.active.count, - recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, - edition: 'CE' + def features_usage_data + features_usage_data_ce + end + + def features_usage_data_ce + { + signup: current_application_settings.signup_enabled?, + ldap: Gitlab.config.ldap.enabled, + gravatar: current_application_settings.gravatar_enabled?, + omniauth: Gitlab.config.omniauth.enabled, + reply_by_email: Gitlab::IncomingEmail.enabled?, + container_registry: Gitlab.config.registry.enabled, + gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled } + end - usage_data + def components_usage_data + { + gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, + git: { version: Gitlab::Git.version }, + database: { adapter: Gitlab::Database.adapter_name, version: Gitlab::Database.version } + } end def services_usage diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 7a94af2f8f1..f200c694562 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,9 +22,9 @@ module Gitlab params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + GL_USERNAME: user&.username, RepoPath: repo_path } - server = { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) @@ -89,6 +89,13 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + params.merge!( + 'GitalyServer' => gitaly_server_hash(repository), + 'GitalyRepository' => repository.gitaly_repository.to_h + ) + end + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -121,10 +128,10 @@ module Gitlab ] end - def send_artifacts_entry(build, path) + def send_artifacts_entry(build, entry) params = { 'Archive' => build.artifacts_file.path, - 'Entry' => Base64.encode64(path.to_s) + 'Entry' => Base64.encode64(entry.to_s) } [ diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb new file mode 100644 index 00000000000..99a82c849e0 --- /dev/null +++ b/lib/google_api/auth.rb @@ -0,0 +1,54 @@ +module GoogleApi + class Auth + attr_reader :access_token, :redirect_uri, :state + + ConfigMissingError = Class.new(StandardError) + + def initialize(access_token, redirect_uri, state: nil) + @access_token = access_token + @redirect_uri = redirect_uri + @state = state + end + + def authorize_url + client.auth_code.authorize_url( + redirect_uri: redirect_uri, + scope: scope, + state: state # This is used for arbitary redirection + ) + end + + def get_token(code) + ret = client.auth_code.get_token(code, redirect_uri: redirect_uri) + return ret.token, ret.expires_at + end + + protected + + def scope + raise NotImplementedError + end + + private + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "google_oauth2" } + end + + def client + return @client if defined?(@client) + + unless config + raise ConfigMissingError + end + + @client = ::OAuth2::Client.new( + config.app_id, + config.app_secret, + site: 'https://accounts.google.com', + token_url: '/o/oauth2/token', + authorize_url: '/o/oauth2/auth' + ) + end + end +end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb new file mode 100644 index 00000000000..a440a3e3562 --- /dev/null +++ b/lib/google_api/cloud_platform/client.rb @@ -0,0 +1,88 @@ +require 'google/apis/container_v1' + +module GoogleApi + module CloudPlatform + class Client < GoogleApi::Auth + DEFAULT_MACHINE_TYPE = 'n1-standard-1'.freeze + SCOPE = 'https://www.googleapis.com/auth/cloud-platform'.freeze + LEAST_TOKEN_LIFE_TIME = 10.minutes + + class << self + def session_key_for_token + :cloud_platform_access_token + end + + def session_key_for_expires_at + :cloud_platform_expires_at + end + + def new_session_key_for_redirect_uri + SecureRandom.hex.tap do |state| + yield session_key_for_redirect_uri(state) + end + end + + def session_key_for_redirect_uri(state) + "cloud_platform_second_redirect_uri_#{state}" + end + end + + def scope + SCOPE + end + + def validate_token(expires_at) + return false unless access_token + return false unless expires_at + + # Making sure that the token will have been still alive during the cluster creation. + return false if token_life_time(expires_at) < LEAST_TOKEN_LIFE_TIME + + true + end + + def projects_zones_clusters_get(project_id, zone, cluster_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_cluster(project_id, zone, cluster_id) + end + + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + request_body = Google::Apis::ContainerV1::CreateClusterRequest.new( + { + "cluster": { + "name": cluster_name, + "initial_node_count": cluster_size, + "node_config": { + "machine_type": machine_type + } + } + } ) + + service.create_cluster(project_id, zone, request_body) + end + + def projects_zones_operations(project_id, zone, operation_id) + service = Google::Apis::ContainerV1::ContainerService.new + service.authorization = access_token + + service.get_zone_operation(project_id, zone, operation_id) + end + + def parse_operation_id(self_link) + m = self_link.match(%r{projects/.*/zones/.*/operations/(.*)}) + m[1] if m + end + + private + + def token_life_time(expires_at) + DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc + end + end + end +end diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb new file mode 100644 index 00000000000..a17ae55910e --- /dev/null +++ b/lib/rspec_flaky/config.rb @@ -0,0 +1,21 @@ +require 'json' + +module RspecFlaky + class Config + def self.generate_report? + ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + end + + def self.suite_flaky_examples_report_path + ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json") + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json") + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json") + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb index f81fb90e870..6be24014d89 100644 --- a/lib/rspec_flaky/flaky_example.rb +++ b/lib/rspec_flaky/flaky_example.rb @@ -9,24 +9,21 @@ module RspecFlaky line: example.line, description: example.description, last_attempts_count: example.attempts, - flaky_reports: 1) + flaky_reports: 0) else super end end - def first_flaky_at - self[:first_flaky_at] || Time.now - end - - def last_flaky_at - Time.now - end + def update_flakiness!(last_attempts_count: nil) + self.first_flaky_at ||= Time.now + self.last_flaky_at = Time.now + self.flaky_reports += 1 + self.last_attempts_count = last_attempts_count if last_attempts_count - def last_flaky_job - return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] - - "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end end def to_h diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..973c95b0212 --- /dev/null +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,37 @@ +require 'json' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def self.from_json(json) + new(JSON.parse(json)) + end + + def initialize(collection = {}) + unless collection.is_a?(Hash) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" + end + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_report + Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys + end + + def -(other) + unless other.respond_to?(:key) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" + end + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index ec2fbd9e36c..4a5bfec9967 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -2,11 +2,15 @@ require 'json' module RspecFlaky class Listener - attr_reader :all_flaky_examples, :new_flaky_examples - - def initialize - @new_flaky_examples = {} - @all_flaky_examples = init_all_flaky_examples + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end def example_passed(notification) @@ -14,29 +18,21 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example_hash = all_flaky_examples[current_example.uid] - - all_flaky_examples[current_example.uid] = - if flaky_example_hash - FlakyExample.new(flaky_example_hash).tap do |ex| - ex.last_attempts_count = current_example.attempts - ex.flaky_reports += 1 - end - else - FlakyExample.new(current_example).tap do |ex| - new_flaky_examples[current_example.uid] = ex - end - end + flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(all_flaky_examples, all_flaky_examples_report_path) + write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) - write_report_file(new_flaky_examples, new_flaky_examples_report_path) + write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end @@ -46,30 +42,23 @@ module RspecFlaky private - def init_all_flaky_examples - return {} unless File.exist?(all_flaky_examples_report_path) + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + unless suite_flaky_examples_json + return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + end - Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + FlakyExamplesCollection.from_json(suite_flaky_examples_json) end - def write_report_file(examples, file_path) - return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + def write_report_file(examples_collection, file_path) + return unless RspecFlaky::Config.generate_report? report_path_dir = File.dirname(file_path) FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - File.write(file_path, JSON.pretty_generate(to_report(examples))) - end - - def all_flaky_examples_report_path - @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/all-report.json") - end - def new_flaky_examples_report_path - @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/new-report.json") + File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 7b486d78cf0..9af21078403 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -5,15 +5,16 @@ module SystemCheck # whitelisted as it may change the SSH client's behaviour dramatically. WHITELIST = %w[ authorized_keys + authorized_keys.lock authorized_keys2 known_hosts ].freeze set_name 'Git user has default SSH configuration?' - set_skip_reason 'skipped (git user is not present or configured)' + set_skip_reason 'skipped (GitLab read-only, or git user is not present / configured)' def skip? - !home_dir || !File.directory?(home_dir) + Gitlab::Database.read_only? || !home_dir || !File.directory?(home_dir) end def check? diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index c388682dfb4..6ee8c8874ec 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%W(#{Gitlab.config.git.bin_path} --version))) end def check? diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index fd82f5f8a4a..08a2c495bd4 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version))) end def check? diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index dee108d987b..e55bea86d3f 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -4,22 +4,17 @@ module SystemCheck set_name 'IMAP server credentials are correct?' def check? - if mailbox_config - begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.starttls if config[:start_tls] - imap.login(config[:email], config[:password]) - connected = true - rescue - connected = false - end + if config + try_connect_imap + else + @error = "#{mail_room_config_path} does not have mailboxes setup" + false end - - connected end def show_error try_fixing_it( + "An error occurred: #{@error.class}: #{@error.message}", 'Check that the information in config/gitlab.yml is correct' ) for_more_information( @@ -30,15 +25,31 @@ module SystemCheck private - def mailbox_config - return @config if @config + def try_connect_imap + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) + true + rescue => error + @error = error + false + end + + def config + @config ||= load_config + end + + def mail_room_config_path + @mail_room_config_path ||= + Rails.root.join('config', 'mail_room.yml').to_s + end - config_path = Rails.root.join('config', 'mail_room.yml').to_s - erb = ERB.new(File.read(config_path)) - erb.filename = config_path + def load_config + erb = ERB.new(File.read(mail_room_config_path)) + erb.filename = mail_room_config_path config_file = YAML.load(erb.result) - @config = config_file[:mailboxes]&.first + config_file.dig(:mailboxes, 0) end end end diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb new file mode 100644 index 00000000000..b8446300f72 --- /dev/null +++ b/lib/system_check/orphans/namespace_check.rb @@ -0,0 +1,54 @@ +module SystemCheck + module Orphans + class NamespaceCheck < SystemCheck::BaseCheck + set_name 'Orphaned namespaces:' + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end + + clear_namespaces! # releases memory when check finishes + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned namespaces for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_namespaces(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + result << namespace + end + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def existing_namespaces + @namespaces ||= Namespace.where(parent: nil).all.pluck(:path) + end + + def clear_namespaces! + @namespaces = nil + end + end + end +end diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb new file mode 100644 index 00000000000..9b6b2429783 --- /dev/null +++ b/lib/system_check/orphans/repository_check.rb @@ -0,0 +1,68 @@ +module SystemCheck + module Orphans + class RepositoryCheck < SystemCheck::BaseCheck + set_name 'Orphaned repositories:' + attr_accessor :orphans + + def multi_check + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + + repositories = disk_repositories(repository_storage['path']) + orphans = (repositories - fetch_repositories(storage_name)) + + print_orphans(orphans, storage_name) + end + end + + private + + def print_orphans(orphans, storage_name) + if orphans.empty? + $stdout.puts "* No orphaned repositories for #{storage_name} storage".color(:green) + return + end + + orphans.each do |orphan| + $stdout.puts " - #{orphan}".color(:red) + end + end + + def disk_repositories(storage_path) + fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| + namespace = File.basename(namespace_path) + next if namespace.eql?('@hashed') + + fetch_disk_repositories(namespace_path).each do |repo| + result << "#{namespace}/#{File.basename(repo)}" + end + end + end + + def fetch_repositories(storage_name) + sql = " + SELECT + CONCAT(n.path, '/', p.path, '.git') repo, + CONCAT(n.path, '/', p.path, '.wiki.git') wiki + FROM projects p + JOIN namespaces n + ON (p.namespace_id = n.id AND + n.parent_id IS NULL) + WHERE (p.repository_storage LIKE ?) + " + + query = ActiveRecord::Base.send(:sanitize_sql_array, [sql, storage_name]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.connection.select_all(query).rows.try(:flatten!) || [] + end + + def fetch_disk_namespaces(storage_path) + Dir.glob(File.join(storage_path, '*')) + end + + def fetch_disk_repositories(namespace_path) + Dir.glob(File.join(namespace_path, '*')) + end + end + end +end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 259a755d724..a42f02a84fd 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -3,8 +3,8 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all frontend assets' task compile: [ 'yarn:check', - 'rake:assets:precompile', 'gettext:po_to_json', + 'rake:assets:precompile', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 654f638c454..dfade1f3885 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -398,6 +398,35 @@ namespace :gitlab do end end + namespace :orphans do + desc 'Gitlab | Check for orphaned namespaces and repositories' + task check: :environment do + warn_user_is_not_gitlab + checks = [ + SystemCheck::Orphans::NamespaceCheck, + SystemCheck::Orphans::RepositoryCheck + ] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned namespaces in the repositories path' + task check_namespaces: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::NamespaceCheck] + + SystemCheck.run('Orphans', checks) + end + + desc 'GitLab | Check for orphaned repositories in the repositories path' + task check_repositories: :environment do + warn_user_is_not_gitlab + checks = [SystemCheck::Orphans::RepositoryCheck] + + SystemCheck.run('Orphans', checks) + end + end + namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 7ccda04a35f..b4d05f5995a 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -4,7 +4,10 @@ namespace :gitlab do task :ee_compat_check, [:branch] => :environment do |_, args| opts = if ENV['CI'] - { branch: ENV['CI_COMMIT_REF_NAME'] } + { + ce_repo: ENV['CI_REPOSITORY_URL'], + branch: ENV['CI_COMMIT_REF_NAME'] + } else unless args[:branch] puts "Must specify a branch as an argument".color(:red) @@ -13,7 +16,10 @@ namespace :gitlab do args end - if Gitlab::EeCompatCheck.new(opts || {}).check + if File.basename(Rails.root) == 'gitlab-ee' + puts "Skipping EE projects" + exit 0 + elsif Gitlab::EeCompatCheck.new(opts || {}).check exit 0 else exit 1 diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 08677a98fc1..8377fe3269d 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -50,6 +50,8 @@ namespace :gitlab do # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. def gitaly_configuration_toml(gitaly_ruby: true) storages = [] address = nil @@ -67,6 +69,11 @@ namespace :gitlab do storages << { name: key, path: val['path'] } end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 42825f29e32..0e6aed32c52 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -79,7 +79,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage_path, + if Gitlab::Shell.new.add_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 4d485108cf6..7f86fd7b45e 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -39,13 +39,19 @@ class GithubImport def import! @project.force_import_start + import_success = false + timings = Benchmark.measure do - Github::Import.new(@project, @options).execute + import_success = Github::Import.new(@project, @options).execute end - puts "Import finished. Timings: #{timings}".color(:green) - - @project.import_finish + if import_success + @project.import_finish + puts "Import finished. Timings: #{timings}".color(:green) + else + puts "Import was not successful. Errors were as follows:" + puts @project.import_error + end end def new_project @@ -53,18 +59,23 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Projects::CreateService.new( + project = Projects::CreateService.new( @current_user, name: name, path: name, description: @repo['description'], namespace_id: namespace.id, visibility_level: visibility_level, - import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"), skip_wiki: @repo['has_wiki'] ).execute + + project.update!( + import_type: 'github', + import_source: @repo['full_name'], + import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + ) + + project end end |