diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2017-11-22 17:07:01 +0800 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2017-11-22 17:07:01 +0800 |
| commit | 663593e87e81d199a5b9a4926d4644ed15416ab0 (patch) | |
| tree | 4e4c331e27bdfe744360e3c620460c04d86f7c7f /lib | |
| parent | 166a2d7a67787d3cf8cebb1e75fc557e2409e669 (diff) | |
| parent | 6369db0196ec7b6e288b16382c95243424a59b62 (diff) | |
| download | gitlab-ce-663593e87e81d199a5b9a4926d4644ed15416ab0.tar.gz | |
Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules
* upstream/master: (126 commits)
Update VERSION to 10.3.0-pre
Update CHANGELOG.md for 10.2.0
default fill color for SVGs
ignore hashed repos (for now) when using `rake gitlab:cleanup:repos`
Use Redis cache for branch existence checks
Update CONTRIBUTING.md: Link definition of done to criteria
Use `make install` for Gitaly setups in non-test environments
FileUploader should check for hashed_storage?(:attachments) to use disk_path
Set the default gitlab-shell timeout to 3 hours
Update composite pipelines index to include "id"
Use arrays in Pipeline#latest_builds_with_artifacts
Fix blank states using old css
Skip confirmation user api
Custom issue tracker
Revert "check for `read_only?` first before seeing if request is disallowed"
add `#with_metadata` scope to remove a N+1 from the notes' API
Fix promoting milestone updating all issuables without milestone
Batchload blobs for diff generation
check for `read_only?` first before seeing if request is disallowed
use `Gitlab::Routing.url_helpers` instead of `Rails.application.routes.url_helpers`
...
Diffstat (limited to 'lib')
77 files changed, 371 insertions, 207 deletions
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c1c0d344917..9aeebc34525 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,9 +6,6 @@ module API module APIGuard extend ActiveSupport::Concern - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze - PRIVATE_TOKEN_PARAM = :private_token - included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -42,7 +39,7 @@ module API # Helper Methods for Grape Endpoint module HelperMethods - include Gitlab::Utils::StrongMemoize + include Gitlab::Auth::UserAuthFinders def find_current_user! user = find_user_from_access_token || find_user_from_warden @@ -53,76 +50,8 @@ module API user end - def access_token - strong_memoize(:access_token) do - find_oauth_access_token || find_personal_access_token - end - end - - def validate_access_token!(scopes: []) - return unless access_token - - case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) - when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when AccessTokenValidationService::EXPIRED - raise ExpiredError - when AccessTokenValidationService::REVOKED - raise RevokedError - end - end - private - def find_user_from_access_token - return unless access_token - - validate_access_token! - - access_token.user || raise(UnauthorizedError) - end - - # Check the Rails session for valid authentication details - def find_user_from_warden - warden.try(:authenticate) if verified_request? - end - - def warden - env['warden'] - end - - # Check if the request is GET/HEAD, or if CSRF token is valid. - def verified_request? - Gitlab::RequestForgeryProtection.verified?(env) - end - - def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) - return unless token - - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = OauthAccessToken.by_token(token) - raise UnauthorizedError unless access_token - - access_token.revoke_previous_refresh_token! - access_token - end - - def find_personal_access_token - token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - return unless token.present? - - # Expiration, revocation and scopes are verified in `find_user_by_access_token` - access_token = PersonalAccessToken.find_by(token: token) - raise UnauthorizedError unless access_token - - access_token - end - - def doorkeeper_request - @doorkeeper_request ||= ActionDispatch::Request.new(env) - end - # An array of scopes that were registered (using `allow_access_with_scope`) # for the current endpoint class. It also returns scopes registered on # `API::API`, since these are meant to apply to all API routes. @@ -145,8 +74,11 @@ module API private def install_error_responders(base) - error_classes = [MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + error_classes = [Gitlab::Auth::MissingTokenError, + Gitlab::Auth::TokenNotFoundError, + Gitlab::Auth::ExpiredError, + Gitlab::Auth::RevokedError, + Gitlab::Auth::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend end @@ -155,25 +87,25 @@ module API proc do |e| response = case e - when MissingTokenError + when Gitlab::Auth::MissingTokenError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new - when TokenNotFoundError + when Gitlab::Auth::TokenNotFoundError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Bad Access Token.") - when ExpiredError + when Gitlab::Auth::ExpiredError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token is expired. You can either do re-authorization or token refresh.") - when RevokedError + when Gitlab::Auth::RevokedError Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( :invalid_token, "Token was revoked. You have to re-authorize from the user.") - when InsufficientScopeError + when Gitlab::Auth::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. Rack::OAuth2::Server::Resource::Bearer::Forbidden.new( @@ -186,22 +118,5 @@ module API end end end - - # - # Exceptions - # - - MissingTokenError = Class.new(StandardError) - TokenNotFoundError = Class.new(StandardError) - ExpiredError = Class.new(StandardError) - RevokedError = Class.new(StandardError) - UnauthorizedError = Class.new(StandardError) - - class InsufficientScopeError < StandardError - attr_reader :scopes - def initialize(scopes) - @scopes = scopes.map { |s| s.try(:name) || s } - end - end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2bc4039b019..38e05074353 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -180,10 +180,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index fc1a3948bb4..8e37ff7f7ce 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -405,7 +405,7 @@ module API begin @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user! } - rescue APIGuard::UnauthorizedError + rescue Gitlab::Auth::UnauthorizedError unauthorized! end end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 0a8f3073a50..dd4f6c41131 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -4,6 +4,7 @@ module API class Absence < Grape::Validations::Base def validate_param!(attr_name, params) return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 282af32ca94..2cae53dba53 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -14,6 +14,7 @@ module API def get_runner_version_from_params return unless params['info'].present? + attributes_for_keys(%w(name version revision platform architecture), params['info']) end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 0b9ab4eeb05..ceaaeca4046 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -33,7 +33,7 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(noteable.notes) + paginate(noteable.notes.with_metadata) .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else @@ -50,7 +50,7 @@ module API end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do noteable = find_project_noteable(noteables_str, params[:noteable_id]) - note = noteable.notes.find(params[:note_id]) + note = noteable.notes.with_metadata.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) if can_read_note diff --git a/lib/api/runners.rb b/lib/api/runners.rb index d3559ef71be..e816fcdd928 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -165,17 +165,20 @@ module API def authenticate_show_runner!(runner) return if runner.is_shared || current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_update_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("No access granted") unless user_can_access_runner?(runner) end def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) @@ -185,6 +188,7 @@ module API forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner is locked") if runner.locked? return if current_user.admin? + forbidden!("No access granted") unless user_can_access_runner?(runner) end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 00eb7c60f16..c736cc32021 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -95,6 +95,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/api/users.rb b/lib/api/users.rb index d80b364bd09..0cd89b1bcf8 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -31,7 +31,6 @@ module API optional :location, type: String, desc: 'The location of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' - optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider @@ -101,6 +100,7 @@ module API requires :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + optional :skip_confirmation, type: Boolean, desc: 'Flag indicating the account is confirmed' at_least_one_of :password, :reset_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' @@ -134,6 +134,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' optional :email, type: String, desc: 'The email of the user' optional :password, type: String, desc: 'The password of the new user' + optional :skip_reconfirmation, type: Boolean, desc: 'Flag indicating the account skips the confirmation by email' optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index be360fbfc0c..0ef26aa696a 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -169,10 +169,12 @@ module API if params[:path] commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb index faa265f3314..c6d9957d452 100644 --- a/lib/api/v3/runners.rb +++ b/lib/api/v3/runners.rb @@ -51,6 +51,7 @@ module API helpers do def authenticate_delete_runner!(runner) return if current_user.admin? + forbidden!("Runner is shared") if runner.is_shared? forbidden!("Runner associated with more than one project") if runner.projects.count > 1 forbidden!("No access granted") unless user_can_access_runner?(runner) diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 0762fc02d70..126ec72248e 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -91,6 +91,7 @@ module API put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet attrs = declared_params(include_missing: false) @@ -113,6 +114,7 @@ module API delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet snippet.destroy no_content! diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9bb8ed913d8..ecb3affbba5 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -86,6 +86,7 @@ module Banzai def save_options return {} unless base_context[:xhtml] + { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end end diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index fb2faae02bc..a19a05e8c0d 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -52,8 +52,10 @@ module Banzai children.each do |child| next if child.text.blank? + node = nodes.shift break unless node == child + filtered_nodes << node end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 4d336068861..8932d4f2905 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -31,6 +31,7 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) next unless can_read_group_reference?(node, user, groups) + visible << node elsif can_read_project_reference?(node) visible << node diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 5cb9adf52b0..0050295eeda 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -149,6 +149,7 @@ module Banzai def self.full_cache_key(cache_key, pipeline_name) return unless cache_key + ["banzai", *cache_key, pipeline_name || :full] end @@ -157,6 +158,7 @@ module Banzai # method. def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key + Rails.cache.__send__(:expanded_key, full_cache_key(cache_key, pipeline_name)) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index ae65653645b..b1949d693ad 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -30,6 +30,7 @@ module DeclarativePolicy policy_class = class_for_class(subject.class) raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class end @@ -84,6 +85,7 @@ module DeclarativePolicy while subject.respond_to?(:declarative_policy_delegate) raise ArgumentError, "circular delegations" if seen.include?(subject.object_id) + seen << subject.object_id subject = subject.declarative_policy_delegate end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index b028169f500..47542194497 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -276,6 +276,7 @@ module DeclarativePolicy # boolean `false` def cache(key, &b) return @cache[key] if cached?(key) + @cache[key] = yield end @@ -291,6 +292,7 @@ module DeclarativePolicy @_conditions[name] ||= begin raise "invalid condition #{name}" unless self.class.conditions.key?(name) + ManifestCondition.new(self.class.conditions[name], self) end end diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index 0804edba016..780d8f707bd 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -3,6 +3,7 @@ module DeclarativePolicy class << self def user_key(user) return '<anonymous>' if user.nil? + id_for(user) end @@ -15,6 +16,7 @@ module DeclarativePolicy def subject_key(subject) return '<nil>' if subject.nil? return subject.inspect if subject.is_a?(Symbol) + "#{subject.class.name}:#{id_for(subject)}" end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index 7cfa82a9a9f..e309244a3b3 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -83,6 +83,7 @@ module DeclarativePolicy def cached_pass?(context) condition = context.condition(@name) return nil unless condition.cached? + condition.pass? end @@ -109,6 +110,7 @@ module DeclarativePolicy def delegated_context(context) policy = context.delegated_policies[@delegate_name] raise MissingDelegate if policy.nil? + policy end @@ -121,6 +123,7 @@ module DeclarativePolicy def cached_pass?(context) condition = delegated_context(context).condition(@name) return nil unless condition.cached? + condition.pass? rescue MissingDelegate false @@ -157,6 +160,7 @@ module DeclarativePolicy def cached_pass?(context) runner = context.runner(@ability) return nil unless runner.cached? + runner.pass? end @@ -258,6 +262,7 @@ module DeclarativePolicy def score(context) return 0 unless cached_pass?(context).nil? + @rules.map { |r| r.score(context) }.inject(0, :+) end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 45ff2ef9ced..77c91817382 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -43,6 +43,7 @@ module DeclarativePolicy # used by Rule::Ability. See #steps_by_score def score return 0 if cached? + steps.map(&:score).inject(0, :+) end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index de391de9059..69d981e8be9 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -8,6 +8,7 @@ class FileSizeValidator < ActiveModel::EachValidator def initialize(options) if range = (options.delete(:in) || options.delete(:within)) raise ArgumentError, ":in and :within must be a Range" unless range.is_a?(Range) + options[:minimum], options[:maximum] = range.begin, range.end options[:maximum] -= 1 if range.exclude_end? end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index b4012ebbb99..7127948cf00 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -58,9 +58,9 @@ module Gitlab def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, - "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE, + "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL } end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb new file mode 100644 index 00000000000..46ec040ce92 --- /dev/null +++ b/lib/gitlab/auth/request_authenticator.rb @@ -0,0 +1,25 @@ +# Use for authentication only, in particular for Rack::Attack. +# Does not perform authorization of scopes, etc. +module Gitlab + module Auth + class RequestAuthenticator + include UserAuthFinders + + attr_reader :request + + def initialize(request) + @request = request + end + + def user + find_sessionless_user || find_user_from_warden + end + + def find_sessionless_user + find_user_from_access_token || find_user_from_rss_token + rescue Gitlab::Auth::AuthenticationError + nil + end + end + end +end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb new file mode 100644 index 00000000000..b4114a3ac96 --- /dev/null +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -0,0 +1,109 @@ +module Gitlab + module Auth + # + # Exceptions + # + + AuthenticationError = Class.new(StandardError) + MissingTokenError = Class.new(AuthenticationError) + TokenNotFoundError = Class.new(AuthenticationError) + ExpiredError = Class.new(AuthenticationError) + RevokedError = Class.new(AuthenticationError) + UnauthorizedError = Class.new(AuthenticationError) + + class InsufficientScopeError < AuthenticationError + attr_reader :scopes + def initialize(scopes) + @scopes = scopes.map { |s| s.try(:name) || s } + end + end + + module UserAuthFinders + include Gitlab::Utils::StrongMemoize + + PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'.freeze + PRIVATE_TOKEN_PARAM = :private_token + + # Check the Rails session for valid authentication details + def find_user_from_warden + current_request.env['warden']&.authenticate if verified_request? + end + + def find_user_from_rss_token + return unless current_request.path.ends_with?('.atom') || current_request.format.atom? + + token = current_request.params[:rss_token].presence + return unless token + + User.find_by_rss_token(token) || raise(UnauthorizedError) + end + + def find_user_from_access_token + return unless access_token + + validate_access_token! + + access_token.user || raise(UnauthorizedError) + end + + def validate_access_token!(scopes: []) + return unless access_token + + case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) + when AccessTokenValidationService::EXPIRED + raise ExpiredError + when AccessTokenValidationService::REVOKED + raise RevokedError + end + end + + private + + def access_token + strong_memoize(:access_token) do + find_oauth_access_token || find_personal_access_token + end + end + + def find_personal_access_token + token = + current_request.params[PRIVATE_TOKEN_PARAM].presence || + current_request.env[PRIVATE_TOKEN_HEADER].presence + + return unless token + + # Expiration, revocation and scopes are verified in `validate_access_token!` + PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) + end + + def find_oauth_access_token + token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + return unless token + + # Expiration, revocation and scopes are verified in `validate_access_token!` + oauth_token = OauthAccessToken.by_token(token) + raise UnauthorizedError unless oauth_token + + oauth_token.revoke_previous_refresh_token! + oauth_token + end + + # Check if the request is GET/HEAD, or if CSRF token is valid. + def verified_request? + Gitlab::RequestForgeryProtection.verified?(current_request.env) + end + + def ensure_action_dispatch_request(request) + return request if request.is_a?(ActionDispatch::Request) + + ActionDispatch::Request.new(request.env) + end + + def current_request + @current_request ||= ensure_action_dispatch_request(request) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 033ecd15749..d48ae17aeaf 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,9 +61,9 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.full_path}.wiki" + disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 5b32fca00a4..9c9e6668e6f 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -16,6 +16,7 @@ module Gitlab @changes ||= begin @raw_changes.map do |change| next if change.blank? + oldrev, newrev, ref = change.strip.split(' ') { oldrev: oldrev, newrev: newrev, ref: ref } end.compact diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index a788fb3fcbc..0bbd60d8ffe 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -98,6 +98,7 @@ module Gitlab def read_string(gz) string_size = read_uint32(gz) return nil unless string_size + gz.read(string_size) end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 22941d48edf..5b2f09e03ea 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -43,6 +43,7 @@ module Gitlab def parent return nil unless has_parent? + self.class.new(@path.to_s.chomp(basename), @entries) end @@ -64,6 +65,7 @@ module Gitlab def directories(opts = {}) return [] unless directory? + dirs = children.select(&:directory?) return dirs unless has_parent? && opts[:parent] @@ -74,6 +76,7 @@ module Gitlab def files return [] unless directory? + children.select(&:file?) end diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index b88b2e36d53..c811f88f483 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -8,6 +8,7 @@ module Gitlab def from_image(job) image = Gitlab::Ci::Build::Image.new(job.options[:image]) return unless image.valid? + image end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 6555c589173..2844be80a84 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -37,6 +37,7 @@ module Gitlab def value return { name: @config } if string? return @config if hash? + {} end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 0159179f0a9..eb606b57667 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -111,6 +111,7 @@ module Gitlab def validate_string_or_regexp(value) return false unless value.is_a?(String) return validate_regexp(value) if look_like_regexp?(value) + true end end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index f07fd1dfdda..633de9f9776 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,6 +2,7 @@ module Gitlab class Daemon def self.initialize_instance(*args) raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) Kernel.at_exit(&@instance.method(:stop)) @instance diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 2c35da8f1aa..c276c3566b4 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -220,6 +220,15 @@ module Gitlab # column - The name of the column to update. # value - The value for the column. # + # The `value` argument is typically a literal. To perform a computed + # update, an Arel literal can be used instead: + # + # update_value = Arel.sql('bar * baz') + # + # update_column_in_batches(:projects, :foo, update_value) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop # determines this method to be too complex while there's no way to make it # less "complex" without introducing extra methods (which actually will diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 403afbe3b9a..fd4a8832ec2 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -68,6 +68,11 @@ module Gitlab has_one :route, as: :source self.table_name = 'projects' + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze + def repository_storage_path Gitlab.config.repositories.storages[repository_storage]['path'] end @@ -76,6 +81,13 @@ module Gitlab def self.name 'Project' end + + def hashed_storage?(feature) + raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature) + return false unless respond_to?(:storage_version) + + self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature] + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 75a75f61953..d32616862f0 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -22,9 +22,11 @@ module Gitlab end def move_project_folders(project, old_full_path, new_full_path) - move_repository(project, old_full_path, new_full_path) - move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") - move_uploads(old_full_path, new_full_path) + unless project.hashed_storage?(:repository) + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ea5891a028a..d0cfe2386ca 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -25,6 +25,10 @@ module Gitlab @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + + # Ensure items are collected in the the batch + new_blob + old_blob end def position(position_marker, position_type: :text) @@ -95,21 +99,15 @@ module Gitlab end def new_blob - return @new_blob if defined?(@new_blob) - - sha = new_content_sha - return @new_blob = nil unless sha + return unless new_content_sha - @new_blob = repository.blob_at(sha, file_path) + Blob.lazy(repository.project, new_content_sha, file_path) end def old_blob - return @old_blob if defined?(@old_blob) - - sha = old_content_sha - return @old_blob = nil unless sha + return unless old_content_sha - @old_blob = repository.blob_at(sha, old_path) + Blob.lazy(repository.project, old_content_sha, old_path) end def content_sha diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 88ae65cb468..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -22,10 +22,7 @@ module Gitlab end def diff_files - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37445 - Gitlab::GitalyClient.allow_n_plus_1_calls do - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } - end + @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end def diff_file_with_old_path(old_path) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 55708d42161..2d7b57120a6 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -102,6 +102,7 @@ module Gitlab new_char = b[pos] break if old_char != new_char + length += 1 end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7dc9cc7c281..8302f30a0a2 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -30,6 +30,7 @@ module Gitlab line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 next if line_old <= 1 && line_new <= 1 # top of file + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 next diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index ccfb908bcca..690b27cde81 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -125,6 +125,7 @@ module Gitlab def find_diff_file(repository) return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) + comparison.diffs(paths: paths, expanded: true).diff_files.first end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 5894384da5d..ea80e21532e 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -16,6 +16,7 @@ module Gitlab noteable = sent_notification.noteable raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) end diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 2152182b37f..acb000e3e23 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -45,6 +45,7 @@ module Gitlab project_name = repo(project_id).name res = @api.command(:search, q: "project:'#{project_name}'", cols: 'ixPersonAssignedTo,ixPersonOpenedBy,ixPersonClosedBy,sStatus,sPriority,sCategory,fOpen,sTitle,sLatestTextSummary,dtOpened,dtClosed,dtResolved,dtLastUpdated,events') return [] unless res['cases']['count'].to_i > 0 + res['cases']['case'] end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 3dcee681c72..5e426b13ade 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -18,6 +18,7 @@ module Gitlab def execute return true unless repo.valid? + client = Gitlab::FogbugzImport::Client.new(token: fb_session[:token], uri: fb_session[:uri]) @cases = client.cases(@repo.id.to_i) @@ -206,6 +207,7 @@ module Gitlab def format_content(raw_content) return raw_content if raw_content.nil? + linkify_issues(escape_for_markdown(raw_content)) end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index cc6c7609ec7..ddd52136bc4 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -102,6 +102,7 @@ module Gitlab if path_arr.size > 1 return nil unless entry[:type] == :tree + path_arr.shift find_entry_by_path(repository, entry[:oid], path_arr.join('/')) else @@ -178,6 +179,8 @@ module Gitlab ) end end + rescue Rugged::ReferenceError + nil end def rugged_raw(repository, sha, limit:) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index cfb88a0c12b..3cb9b254e6e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -304,7 +304,13 @@ module Gitlab end def delete_all_refs_except(prefixes) - delete_refs(*all_ref_names_except(prefixes)) + gitaly_migrate(:ref_delete_refs) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) + else + delete_refs(*all_ref_names_except(prefixes)) + end + end end # Returns an Array of all ref names, except when it's matching pattern @@ -984,6 +990,10 @@ module Gitlab @attributes.attributes(path) end + def gitattribute(path, name) + attributes(path)[name] + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled @@ -1151,6 +1161,11 @@ module Gitlab Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end + # Items should be of format [[commit_id, path], [commit_id1, path1]] + def batch_blobs(items, blob_size_limit: nil) + Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) + end + def commit_index(user, branch_name, index, options) committer = user_to_committer(user) @@ -1376,6 +1391,7 @@ module Gitlab end return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] end end @@ -1496,6 +1512,7 @@ module Gitlab # Ref names must start with `refs/`. def rugged_ref_exists?(ref_name) raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) rescue Rugged::ReferenceError false @@ -1562,6 +1579,7 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 637e7a0659c..4500482d68f 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -78,7 +78,7 @@ module Gitlab def list_remote_tags(remote) tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| tag_list = stdout.read @@ -88,7 +88,7 @@ module Gitlab raise RemoteError, error unless exit_code.zero? - tag_list.split('\n') + tag_list.split("\n") end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 022d1f249a9..d4a53d32c28 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -58,12 +58,12 @@ module Gitlab end end - def pages - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| + def pages(limit: nil) + @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_get_all_pages else - gollum_get_all_pages + gollum_get_all_pages(limit: limit) end end end @@ -88,14 +88,23 @@ module Gitlab end end - def page_versions(page_path) + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options = {}) current_page = gollum_page_by_path(page_path) - current_page.versions.map do |gollum_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) - new_version(gollum_page, gollum_git_commit.id) + + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) end end + def count_page_versions(page_path) + @repository.count_commits(ref: 'HEAD', path: page_path) + end + def preview_slug(title, format) # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid # using Rugged through a Gollum::Wiki instance @@ -110,6 +119,22 @@ module Gitlab private + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def commits_from_page(gollum_page, options = {}) + unless options[:limit] + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + end + + @repository.log(ref: gollum_page.last_version.id, + path: gollum_page.path, + limit: options[:limit], + offset: options[:offset]) + end + def gollum_wiki @gollum_wiki ||= Gollum::Wiki.new(@repository.path) end @@ -126,8 +151,17 @@ module Gitlab end def new_version(gollum_page, commit_id) - commit = Gitlab::Git::Commit.find(@repository, commit_id) - Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) + end + + def version(commit_id) + commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } + + if RequestStore.active? + RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } + else + commit_find_proc.call + end end def assert_type!(object, klass) @@ -185,8 +219,8 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end - def gollum_get_all_pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + def gollum_get_all_pages(limit: nil) + gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } end def gitaly_write_page(name, format, content, commit_details) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index b0c73395cb1..31b04bc2650 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -126,6 +126,15 @@ module Gitlab GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end + def delete_refs(except_with_prefixes:) + request = Gitaly::DeleteRefsRequest.new( + repository: @gitaly_repo, + except_with_prefix: except_with_prefixes + ) + + GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + end + private def consume_refs_response(response) @@ -137,6 +146,7 @@ module Gitlab enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 8f05f40365e..c8f065f5881 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -94,6 +94,7 @@ module Gitlab page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } break unless page && version + pages << [page, version] end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index f1007daab5d..075b3982608 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -65,6 +65,7 @@ module Gitlab y << item end break if items.empty? || items.size < per_page + page += 1 end end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 61db4bd9ccc..f3d7407383c 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -1,7 +1,7 @@ module Gitlab module ImportExport class MergeRequestParser - FORKED_PROJECT_ID = -1 + FORKED_PROJECT_ID = nil def initialize(project, diff_head_sha, merge_request, relation_hash) @project = project diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index c8479fbc0e8..fbbddb7bffa 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -12,6 +12,7 @@ module Gitlab @client.get_namespace(name) rescue ::KubeException => ke raise ke unless ke.error_code == 404 + false end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index ed1de73f8c6..7274d1c3b43 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -62,6 +62,7 @@ module Gitlab def user return nil unless ldap_user + Gitlab::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) end end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 4d5c67ed892..3945df27eed 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -9,11 +9,8 @@ module Gitlab class User < Gitlab::OAuth::User class << self def find_by_uid_and_provider(uid, provider) - uid = Gitlab::LDAP::Person.normalize_dn(uid) + identity = ::Identity.with_extern_uid(provider, uid).take - identity = ::Identity - .where(provider: provider) - .where(extern_uid: uid).last identity && identity.user end end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 12c968805f5..4d096e5a741 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -15,6 +15,7 @@ module Gitlab def client return @client if defined?(@client) + unless credentials raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8b5a60e6b8b..436a9e9550d 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -96,6 +96,7 @@ module Gitlab def worker_label return {} unless defined?(Unicorn::Worker) + worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 064299f40c8..ead1acb8d44 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -7,6 +7,7 @@ module Gitlab def sql(event) return unless current_transaction + metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index cfc6b2a2029..c6a56277922 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -42,12 +42,11 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = case current_application_settings.enabled_git_access_protocol - when 'ssh' + repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" - when 'http', nil + else "#{project_url}.git" end @@ -66,6 +65,7 @@ module Gitlab project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match + path = project_path_match[1] # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 5e4932e4e57..c26656704d7 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -58,7 +58,7 @@ module Gitlab end def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Rails.application.routes.url_helpers.root_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url end def route_hash @@ -74,10 +74,16 @@ module Gitlab end def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' end def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' end end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index eb3c9002710..c22d0a84860 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -55,7 +55,9 @@ module Gitlab def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) - @first_collection_last_page_size = paginated_first_collection(first_collection_page_count).count + @first_collection_last_page_size = paginated_first_collection(first_collection_page_count) + .except(:select) + .size end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b4b3b00c84d..552133234a3 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -157,7 +157,7 @@ module Gitlab end def find_by_uid_and_provider - identity = Identity.find_by(provider: auth_hash.provider, extern_uid: auth_hash.uid) + identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity && identity.user end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 962ff4d3985..1d9a5d1a20a 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -11,6 +11,7 @@ module Gitlab rescue ActiveRecord::StaleObjectError retries -= 1 raise unless retries >= 0 + subject.reload end end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 910533076b0..2c994536060 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -46,10 +46,10 @@ module Gitlab # Only replace the last occurence of `path`. # # `request.fullpath` includes the querystring - path = request.path.sub(%r{/#{path}/*(?!.*#{path})}, "/-/#{path}/") - path << "?#{request.query_string}" if request.query_string.present? + new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") + new_path << "?#{request.query_string}" if request.query_string.present? - path + new_path end paths.each do |path| diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index e0a9d1dee77..d8faf7aad8c 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -28,6 +28,7 @@ module Gitlab def changed? return true unless gl_user + gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a37112ae5c4..996d213fdb4 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -101,8 +101,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) - # Timeout should be less than 900 ideally, to prevent the memory killer - # to silently kill the process without knowing we are timing out here. + # The timeout ensures the subprocess won't hang forever cmd = [gitlab_shell_projects_path, 'import-project', storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] gitlab_shell_fast_execute_raise_error(cmd) @@ -368,6 +367,7 @@ module Gitlab output, status = gitlab_shell_fast_execute_helper(cmd, vars) raise Error, output unless status.zero? + true end diff --git a/lib/gitlab/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..053dd4ab9e0 100644 --- a/lib/gitlab/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb @@ -5,7 +5,7 @@ module Gitlab module ShellAdapter def gitlab_shell - Gitlab::Shell.new + @gitlab_shell ||= Gitlab::Shell.new end end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 11aeec1ebfa..f9faa134206 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -90,6 +90,7 @@ module Gitlab # Takes an array of integers, and returns an array of ranges covering the same integers def collapse_ranges(positions) return [] if positions.empty? + ranges = [] start = prev = positions[0] diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index cb7957e2af9..33f07fa0120 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -18,6 +18,7 @@ module Gitlab def read(path) blob = @repository.blob_at(@commit.id, path) if @commit raise FileNotFoundError if blob.nil? + blob.data end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 1caa791c1be..59331c827af 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -70,6 +70,7 @@ module Gitlab def generate_full_url return @url unless valid_credentials? + @full_url = @url.dup @full_url.password = credentials[:password] if credentials[:password].present? diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index c60bd91ea6e..11472ce6cce 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -99,6 +99,7 @@ module Gitlab def level_value(level) return level.to_i if level.to_i.to_s == level.to_s && string_options.key(level.to_i) + string_options[level] || PRIVATE end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e1219df1b25..864a9e04888 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -174,6 +174,7 @@ module Gitlab @secret ||= begin bytes = Base64.strict_decode64(File.read(secret_path).chomp) raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes end end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index 05668c69006..f5485eb89fa 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -9,6 +9,7 @@ unless Rails.env.production? def visit_filter(node) return unless node.filter_type == 'javascript' + record_lint(node, 'Inline JavaScript is discouraged (https://docs.gitlab.com/ee/development/gotchas.html#do-not-use-inline-javascript-in-views)') end end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 00221f77cf4..8b145fb4511 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -24,6 +24,7 @@ module SystemCheck # @param [BaseCheck] check class def <<(check) raise ArgumentError unless check.is_a?(Class) && check < BaseCheck + @checks << check end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 8ae1b6a626a..301affc9522 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -59,7 +59,11 @@ namespace :gitlab do .sub(%r{^/*}, '') .chomp('.git') .chomp('.wiki') - next if Project.find_by_full_path(repo_with_namespace) + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?('@hashed/') || Project.find_by_full_path(repo_with_namespace) + new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect File.rename(path, new_path) @@ -75,6 +79,7 @@ namespace :gitlab do User.find_each do |user| next unless user.ldap_user? + print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 8377fe3269d..f2002d7a426 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -14,18 +14,18 @@ namespace :gitlab do checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) + command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + _, status = Gitlab::Popen.popen(%w[which gmake]) - command = status.zero? ? ['gmake'] : ['make'] + command << (status.zero? ? 'gmake' : 'make') - if Rails.env.test? - command += %W[BUNDLE_PATH=#{Bundler.bundle_path}] - end + command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? Dir.chdir(args.dir) do create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) } + Bundler.with_original_env { run_command!(command) } end end end @@ -82,9 +82,12 @@ namespace :gitlab do end def create_gitaly_configuration - File.open("config.toml", "w") do |f| + File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| f.puts gitaly_configuration_toml end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." rescue ArgumentError => e puts "Skipping config.toml generation:" puts e.message diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake deleted file mode 100644 index 6cbc83b8973..00000000000 --- a/lib/tasks/gitlab/sidekiq.rake +++ /dev/null @@ -1,47 +0,0 @@ -namespace :gitlab do - namespace :sidekiq do - QUEUE = 'queue:post_receive'.freeze - - desc 'Drop all Sidekiq PostReceive jobs for a given project' - task :drop_post_receive, [:project] => :environment do |t, args| - unless args.project.present? - abort "Please specify the project you want to drop PostReceive jobs for:\n rake gitlab:sidekiq:drop_post_receive[group/project]" - end - project_path = Project.find_by_full_path(args.project).repository.path_to_repo - - Sidekiq.redis do |redis| - unless redis.exists(QUEUE) - abort "Queue #{QUEUE} is empty" - end - - temp_queue = "#{QUEUE}_#{Time.now.to_i}" - redis.rename(QUEUE, temp_queue) - - # At this point, then post_receive queue is empty. It may be receiving - # new jobs already. We will repopulate it with the old jobs, skipping the - # ones we want to drop. - dropped = 0 - while (job = redis.lpop(temp_queue)) - if repo_path(job) == project_path - dropped += 1 - else - redis.rpush(QUEUE, job) - end - end - # The temp_queue will delete itself after we have popped all elements - # from it - - puts "Dropped #{dropped} jobs containing #{project_path} from #{QUEUE}" - end - end - - def repo_path(job) - job_args = JSON.parse(job)['args'] - if job_args - job_args.first - else - nil - end - end - end -end |