diff options
Diffstat (limited to 'lib')
94 files changed, 1984 insertions, 698 deletions
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c4c0fdda665..e79f988f549 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -75,7 +75,7 @@ module API raise RevokedError when AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) + User.find(access_token.resource_owner_id) end end @@ -84,11 +84,13 @@ module API return nil unless token_string.present? - find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) - end + user = + find_user_by_authentication_token(token_string) || + find_user_by_personal_access_token(token_string, scopes) + + raise UnauthorizedError unless user - def current_user - @current_user + user end private @@ -107,7 +109,16 @@ module API end def find_access_token - @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) + return @access_token if defined?(@access_token) + + token = Doorkeeper::OAuth::Token.from_request(doorkeeper_request, *Doorkeeper.configuration.access_token_methods) + return @access_token = nil unless token + + @access_token = Doorkeeper::AccessToken.by_token(token) + raise UnauthorizedError unless @access_token + + @access_token.revoke_previous_refresh_token! + @access_token end def doorkeeper_request @@ -169,6 +180,7 @@ module API TokenNotFoundError = Class.new(StandardError) ExpiredError = Class.new(StandardError) RevokedError = Class.new(StandardError) + UnauthorizedError = Class.new(StandardError) class InsufficientScopeError < StandardError attr_reader :scopes diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb new file mode 100644 index 00000000000..5000aa0d9ac --- /dev/null +++ b/lib/api/custom_attributes_endpoints.rb @@ -0,0 +1,77 @@ +module API + module CustomAttributesEndpoints + extend ActiveSupport::Concern + + included do + attributable_class = name.demodulize.singularize + attributable_key = attributable_class.underscore + attributable_name = attributable_class.humanize(capitalize: false) + attributable_finder = "find_#{attributable_key}" + + helpers do + params :custom_attributes_key do + requires :key, type: String, desc: 'The key of the custom attribute' + end + end + + desc "Get all custom attributes on a #{attributable_name}" do + success Entities::CustomAttribute + end + get ':id/custom_attributes' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + present resource.custom_attributes, with: Entities::CustomAttribute + end + + desc "Get a custom attribute on a #{attributable_name}" do + success Entities::CustomAttribute + end + params do + use :custom_attributes_key + end + get ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :read_custom_attribute + + custom_attribute = resource.custom_attributes.find_by!(key: params[:key]) + + present custom_attribute, with: Entities::CustomAttribute + end + + desc "Set a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + requires :value, type: String, desc: 'The value of the custom attribute' + end + put ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + custom_attribute = resource.custom_attributes + .find_or_initialize_by(key: params[:key]) + + custom_attribute.update(value: params[:value]) + + if custom_attribute.valid? + present custom_attribute, with: Entities::CustomAttribute + else + render_validation_error!(custom_attribute) + end + end + + desc "Delete a custom attribute on a #{attributable_name}" + params do + use :custom_attributes_key + end + delete ':id/custom_attributes/:key' do + resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend + authorize! :update_custom_attribute + + resource.custom_attributes.find_by!(key: params[:key]).destroy + + status 204 + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 71253f72533..7082f31b5b8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -89,6 +89,9 @@ module API expose :ssh_url_to_repo, :http_url_to_repo, :web_url expose :name, :name_with_namespace expose :path, :path_with_namespace + expose :avatar_url do |project, options| + project.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :created_at, :last_activity_at end @@ -146,9 +149,7 @@ module API expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) - end + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds, as: :public_jobs @@ -193,8 +194,8 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url do |user, options| - user.avatar_url(only_path: false) + expose :avatar_url do |group, options| + group.avatar_url(only_path: false) end expose :web_url expose :request_access_enabled @@ -234,6 +235,7 @@ module API class RepoCommitDetail < RepoCommit expose :stats, using: Entities::RepoCommitStats expose :status + expose :last_pipeline, using: 'API::Entities::PipelineBasic' end class RepoBranch < Grape::Entity @@ -1020,6 +1022,7 @@ module API expose :cache, using: Cache expose :credentials, using: Credentials expose :dependencies, using: Dependency + expose :features end end @@ -1034,5 +1037,10 @@ module API expose :failing_on_hosts expose :total_failures end + + class CustomAttribute < Grape::Entity + expose :key + expose :value + end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 00dbc2aee7a..4964a76bef6 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -3,6 +3,8 @@ module API include Gitlab::Utils include Helpers::Pagination + UnauthorizedError = Class.new(StandardError) + SUDO_HEADER = "HTTP_SUDO".freeze SUDO_PARAM = :sudo @@ -139,7 +141,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(initial_current_user, :access_api) + unauthorized! unless current_user end def authenticate_non_get! @@ -397,19 +399,27 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) - Gitlab::Auth::UniqueIpsLimiter.limit_user! do - @initial_current_user ||= find_user_by_private_token(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= doorkeeper_guard(scopes: scopes_registered_for_endpoint) - @initial_current_user ||= find_user_from_warden - - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil - end - @initial_current_user + begin + @initial_current_user = Gitlab::Auth::UniqueIpsLimiter.limit_user! { find_current_user } + rescue APIGuard::UnauthorizedError, UnauthorizedError + unauthorized! end end + def find_current_user + user = + find_user_by_private_token(scopes: scopes_registered_for_endpoint) || + doorkeeper_guard(scopes: scopes_registered_for_endpoint) || + find_user_from_warden + + return nil unless user + + raise UnauthorizedError unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) + + user + end + def sudo! return unless sudo_identifier return unless initial_current_user @@ -454,10 +464,12 @@ module API header(*Gitlab::Workhorse.send_artifacts_entry(build, entry)) end - # The Grape Error Middleware only has access to env but no params. We workaround this by - # defining a method that returns the right value. + # The Grape Error Middleware only has access to `env` but not `params` nor + # `request`. We workaround this by defining methods that returns the right + # values. def define_params_for_grape_middleware - self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:params) { request.params.symbolize_keys } end # We could get a Grape or a standard Ruby exception. We should only report anything that diff --git a/lib/api/internal.rb b/lib/api/internal.rb index c0fef56378f..6e78ac2c903 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -31,6 +31,12 @@ module API protocol = params[:protocol] actor.update_last_used_at if actor.is_a?(Key) + user = + if actor.is_a?(Key) + actor.user + else + actor + end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess access_checker = access_checker_klass @@ -47,6 +53,7 @@ module API { status: true, gl_repository: gl_repository, + gl_username: user&.username, repository_path: repository_path, gitaly: gitaly_payload(params[:action]) } @@ -136,7 +143,7 @@ module API codes = nil - ::Users::UpdateService.new(user).execute! do |user| + ::Users::UpdateService.new(current_user, user: user).execute! do |user| codes = user.generate_otp_backup_codes! end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bcc0833aa5c..0266bf2f717 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -35,7 +35,7 @@ module API new_notification_email = params.delete(:notification_email) if new_notification_email - ::Users::UpdateService.new(current_user, notification_email: new_notification_email).execute + ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute end notification_setting.update(declared_params(include_missing: false)) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7dc19788462..aab7a6c3f93 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -70,8 +70,11 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(options = {}) - projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + def load_projects + ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + end + + def present_projects(projects, options = {}) projects = reorder_projects(projects) projects = projects.with_statistics if params[:statistics] projects = projects.with_issues_enabled if params[:with_issues_enabled] @@ -111,7 +114,7 @@ module API params[:user] = user - present_projects + present_projects load_projects end end @@ -124,7 +127,7 @@ module API use :statistics_params end get do - present_projects + present_projects load_projects end desc 'Create new project' do @@ -229,6 +232,18 @@ module API end end + desc 'List forks of this project' do + success Entities::Project + end + params do + use :collection_params + end + get ':id/forks' do + forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute + + present_projects forks + end + desc 'Update an existing project' do success Entities::Project end diff --git a/lib/api/users.rb b/lib/api/users.rb index bdebda58d3f..d07dc302717 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,12 +6,14 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + include CustomAttributesEndpoints + before do authenticate_non_get! end helpers do - def find_user(params) + def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end @@ -166,7 +168,7 @@ module API user_params[:password_expires_at] = Time.now if user_params[:password].present? - result = ::Users::UpdateService.new(user, user_params.except(:extern_uid, :provider)).execute + result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute if result[:status] == :success present user, with: Entities::UserPublic @@ -326,7 +328,7 @@ module API user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - email = Emails::CreateService.new(user, declared_params(include_missing: false)).execute + email = Emails::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute if email.errors.blank? NotificationService.new.new_email(email) @@ -367,7 +369,7 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: user, email: email.email).execute end user.update_secondary_emails! @@ -430,7 +432,7 @@ module API resource :impersonation_tokens do helpers do def finder(options = {}) - user = find_user(params) + user = find_user_by_id(params) PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end @@ -672,7 +674,7 @@ module API requires :email, type: String, desc: 'The new email' end post "emails" do - email = Emails::CreateService.new(current_user, declared_params).execute + email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? NotificationService.new.new_email(email) @@ -691,7 +693,7 @@ module API not_found!('Email') unless email destroy_conditionally!(email) do |email| - Emails::DestroyService.new(current_user, email: email.email).execute + Emails::DestroyService.new(current_user, user: current_user, email: email.email).execute end current_user.update_secondary_emails! diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index b9a573d3542..3cf3939994a 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -79,7 +79,7 @@ module Backup # - 1495527122_gitlab_backup.tar # - 1495527068_2017_05_23_gitlab_backup.tar # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar - next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+.*)?)?_gitlab_backup\.tar$/ + next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/ timestamp = $1.to_i diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4e92be85110..3ad09a1b421 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -78,7 +78,7 @@ module Backup project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) else %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index ee73fa91589..9cac303e645 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,6 +1,18 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + REDCARPET_OPTIONS = { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + def initialize(text, context = nil, result = nil) super text, context, result @text = @text.delete "\r" @@ -13,27 +25,11 @@ module Banzai end def self.renderer - @renderer ||= begin + Thread.current[:banzai_markdown_renderer] ||= begin renderer = Banzai::Renderer::HTML.new - Redcarpet::Markdown.new(renderer, redcarpet_options) + Redcarpet::Markdown.new(renderer, REDCARPET_OPTIONS) end end - - def self.redcarpet_options - # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use - @redcarpet_options ||= { - fenced_code_blocks: true, - footnotes: true, - lax_spacing: true, - no_intra_emphasis: true, - space_after_headers: true, - strikethrough: true, - superscript: true, - tables: true - }.freeze - end - - private_class_method :redcarpet_options end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 9923ec4e870..d8c8deea628 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -45,8 +45,9 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) - # Disallow `name` attribute globally + # Disallow `name` attribute globally, allow on `a` whitelist[:attributes][:all].delete('name') + whitelist[:attributes]['a'].push('name') # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') @@ -72,8 +73,9 @@ module Banzai return unless node.has_attribute?('href') begin + node['href'] = node['href'].strip uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + uri.scheme = uri.scheme.downcase if uri.scheme node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) rescue Addressable::URI::InvalidURIError diff --git a/lib/github/client.rb b/lib/github/client.rb index 9c476df7d46..29bd9c1f39e 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,6 +1,7 @@ module Github class Client TIMEOUT = 60 + DEFAULT_PER_PAGE = 100 attr_reader :connection, :rate_limit @@ -20,7 +21,7 @@ module Github exceed, reset_in = rate_limit.get sleep reset_in if exceed - Github::Response.new(connection.get(url, query)) + Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) end private diff --git a/lib/github/import.rb b/lib/github/import.rb index 9354e142d3d..c0cd8382875 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,46 +1,13 @@ require_relative 'error' +require_relative 'import/issue' +require_relative 'import/legacy_diff_note' +require_relative 'import/merge_request' +require_relative 'import/note' module Github class Import include Gitlab::ShellAdapter - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class Note < ::Note - self.table_name = 'notes' - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, :options, :errors, :cached, :verbose @@ -202,13 +169,8 @@ module Github merge_request.save!(validate: false) merge_request.merge_request_diffs.create - # Fetch review comments review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - - # Fetch comments - comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" - fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) ensure @@ -241,12 +203,17 @@ module Github # for both features, like manipulating assignees, labels # and milestones, are provided within the Issues API. if representation.pull_request? - return unless representation.has_labels? + return unless representation.has_labels? || representation.has_comments? merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + + if representation.has_labels? + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + end + + fetch_comments_conditionally(merge_request, representation) else - return if Issue.where(iid: representation.iid, project_id: project.id).exists? + return if Issue.exists?(iid: representation.iid, project_id: project.id) author_id = user_id(representation.author, project.creator_id) issue = Issue.new @@ -263,17 +230,20 @@ module Github issue.updated_at = representation.updated_at issue.save!(validate: false) - # Fetch comments - if representation.has_comments? - comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" - fetch_comments(issue, :comment, comments_url) - end + fetch_comments_conditionally(issue, representation) end rescue => e error(:issue, representation.url, e.message) end end + def fetch_comments_conditionally(issuable, representation) + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" + fetch_comments(issuable, :comment, comments_url) + end + end + def fetch_comments(noteable, type, url, klass = Note) while url comments = Github::Client.new(options).get(url) diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb new file mode 100644 index 00000000000..171f0872666 --- /dev/null +++ b/lib/github/import/issue.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :create + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb new file mode 100644 index 00000000000..18adff560b6 --- /dev/null +++ b/lib/github/import/legacy_diff_note.rb @@ -0,0 +1,12 @@ +module Github + class Import + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb new file mode 100644 index 00000000000..c258e5d5e0e --- /dev/null +++ b/lib/github/import/merge_request.rb @@ -0,0 +1,13 @@ +module Github + class Import + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :create + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb new file mode 100644 index 00000000000..8cf4f30e6b7 --- /dev/null +++ b/lib/github/import/note.rb @@ -0,0 +1,13 @@ +module Github + class Import + class Note < ::Note + self.table_name = 'notes' + self.store_full_sti_class = false + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + end +end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 3fde1b09efb..8e5c95f2287 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -3,11 +3,18 @@ module Gitlab class DeserializeMergeRequestDiffsAndCommits attr_reader :diff_ids, :commit_rows, :file_rows + class Error < StandardError + def backtrace + cause.backtrace + end + end + class MergeRequestDiff < ActiveRecord::Base self.table_name = 'merge_request_diffs' end BUFFER_ROWS = 1000 + DIFF_FILE_BUFFER_ROWS = 100 def perform(start_id, stop_id) merge_request_diffs = MergeRequestDiff @@ -26,13 +33,17 @@ module Gitlab if diff_ids.length > BUFFER_ROWS || commit_rows.length > BUFFER_ROWS || - file_rows.length > BUFFER_ROWS + file_rows.length > DIFF_FILE_BUFFER_ROWS flush_buffers! end end flush_buffers! + rescue => e + Rails.logger.info("#{self.class.name}: failed for IDs #{merge_request_diffs.map(&:id)} with #{e.class.name}") + + raise Error.new(e.inspect) end private @@ -45,17 +56,28 @@ module Gitlab def flush_buffers! if diff_ids.any? - MergeRequestDiff.transaction do - Gitlab::Database.bulk_insert('merge_request_diff_commits', commit_rows) - Gitlab::Database.bulk_insert('merge_request_diff_files', file_rows) + commit_rows.each_slice(BUFFER_ROWS).each do |commit_rows_slice| + bulk_insert('merge_request_diff_commits', commit_rows_slice) + end - MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) + file_rows.each_slice(DIFF_FILE_BUFFER_ROWS).each do |file_rows_slice| + bulk_insert('merge_request_diff_files', file_rows_slice) end + + MergeRequestDiff.where(id: diff_ids).update_all(st_commits: nil, st_diffs: nil) end reset_buffers! end + def bulk_insert(table, rows) + Gitlab::Database.bulk_insert(table, rows) + rescue ActiveRecord::RecordNotUnique + ids = rows.map { |row| row[:merge_request_diff_id] }.uniq.sort + + Rails.logger.info("#{self.class.name}: rows inserted twice for IDs #{ids}") + end + def single_diff_rows(merge_request_diff) sha_attribute = Gitlab::Database::ShaAttribute.new commits = YAML.load(merge_request_diff.st_commits) rescue [] diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 9323bfc7fb2..1d98d187805 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -56,7 +56,8 @@ module Gitlab name: project_path, path: project_path, repository_storage: storage_name, - namespace_id: group&.id + namespace_id: group&.id, + skip_disk_validation: true } project = Projects::CreateService.new(user, project_params).execute diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 28bbf3b384e..d1979bb7ed3 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -149,16 +149,21 @@ module Gitlab description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) description += pull_request.description + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha + merge_request = project.merge_requests.create!( iid: pull_request.iid, title: pull_request.title, description: description, source_project: project, source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, + source_branch_sha: source_branch_sha, target_project: project, target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, + target_branch_sha: target_branch_sha, state: pull_request.state, author_id: gitlab_user_id(project, pull_request.author), assignee_id: nil, diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index ad78ae244b2..088adbdd267 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -155,7 +155,9 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) + if s.scan(/section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/) + handle_section(s) + elsif s.scan(/\e([@-_])(.*?)([@-~])/) handle_sequence(s) elsif s.scan(/\e(([@-_])(.*?)?)?$/) break @@ -183,6 +185,15 @@ module Gitlab ) end + def handle_section(s) + action = s[1] + timestamp = s[2] + section = s[3] + line = s.matched()[0...-5] # strips \r\033[0K + + @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} + end + def handle_sequence(s) indicator = s[1] commands = s[2].split ';' diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb new file mode 100644 index 00000000000..d10cc7802d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Build + module Policy + def self.fabricate(specs) + specifications = specs.to_h.map do |spec, value| + self.const_get(spec.to_s.camelize).new(value) + end + + specifications.compact + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb new file mode 100644 index 00000000000..b20d374288f --- /dev/null +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Build + module Policy + class Kubernetes < Policy::Specification + def initialize(spec) + unless spec.to_sym == :active + raise UnknownPolicyError + end + end + + def satisfied_by?(pipeline) + pipeline.has_kubernetes_active? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb new file mode 100644 index 00000000000..eadc0948d2f --- /dev/null +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -0,0 +1,43 @@ +module Gitlab + module Ci + module Build + module Policy + class Refs < Policy::Specification + def initialize(refs) + @patterns = Array(refs) + end + + def satisfied_by?(pipeline) + @patterns.any? do |pattern| + pattern, path = pattern.split('@', 2) + + matches_path?(path, pipeline) && + matches_pattern?(pattern, pipeline) + end + end + + private + + def matches_path?(path, pipeline) + return true unless path + + pipeline.project_full_path == path + end + + def matches_pattern?(pattern, pipeline) + return true if pipeline.tag? && pattern == 'tags' + return true if pipeline.branch? && pattern == 'branches' + return true if pipeline.source == pattern + return true if pipeline.source&.pluralize == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ pipeline.ref + else + pattern == pipeline.ref + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb new file mode 100644 index 00000000000..c317291f29d --- /dev/null +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Build + module Policy + ## + # Abstract class that defines an interface of job policy + # specification. + # + # Used for job's only/except policy configuration. + # + class Specification + UnknownPolicyError = Class.new(StandardError) + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb new file mode 100644 index 00000000000..8d82e1b288d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Base + attr_reader :pipeline, :project, :current_user + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + + @project = command.project + @current_user = command.current_user + end + + def perform! + raise NotImplementedError + end + + def break? + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb new file mode 100644 index 00000000000..d5e17a123df --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Create < Chain::Base + include Chain::Helpers + + def perform! + ::Ci::Pipeline.transaction do + pipeline.save! + + @command.seeds_block&.call(pipeline) + + ::Ci::CreatePipelineStagesService + .new(project, current_user) + .execute(pipeline) + end + rescue ActiveRecord::RecordInvalid => e + error("Failed to persist the pipeline: #{e}") + end + + def break? + !pipeline.persisted? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb new file mode 100644 index 00000000000..02d81286f21 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -0,0 +1,25 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Helpers + def branch_exists? + return @is_branch if defined?(@is_branch) + + @is_branch = project.repository.branch_exists?(pipeline.ref) + end + + def tag_exists? + return @is_tag if defined?(@is_tag) + + @is_tag = project.repository.tag_exists?(pipeline.ref) + end + + def error(message) + pipeline.errors.add(:base, message) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb new file mode 100644 index 00000000000..015f2988327 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -0,0 +1,36 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Sequence + def initialize(pipeline, command, sequence) + @pipeline = pipeline + @completed = [] + + @sequence = sequence.map do |chain| + chain.new(pipeline, command) + end + end + + def build! + @sequence.each do |step| + step.perform! + + break if step.break? + + @completed << step + end + + @pipeline.tap do + yield @pipeline, self if block_given? + end + end + + def complete? + @completed.size == @sequence.size + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb new file mode 100644 index 00000000000..9a72de87bab --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Skip < Chain::Base + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + + def perform! + if skipped? + @pipeline.skip if @command.save_incompleted + end + end + + def skipped? + !@command.ignore_skip_ci && commit_message_skips_ci? + end + + def break? + skipped? + end + + private + + def commit_message_skips_ci? + return false unless @pipeline.git_commit_message + + @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb new file mode 100644 index 00000000000..4913a604079 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Abilities < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + + def perform! + unless project.builds_enabled? + return error('Pipelines are disabled!') + end + + unless allowed_to_trigger_pipeline? + if can?(current_user, :create_pipeline, project) + return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + else + return error('Insufficient permissions to create a new pipeline') + end + end + end + + def break? + @pipeline.errors.any? + end + + def allowed_to_trigger_pipeline? + if current_user + allowed_to_create? + else # legacy triggers don't have a corresponding user + !project.protected_for?(@pipeline.ref) + end + end + + def allowed_to_create? + return unless can?(current_user, :create_pipeline, project) + + access = Gitlab::UserAccess.new(current_user, project: project) + + if branch_exists? + access.can_update_branch?(@pipeline.ref) + elsif tag_exists? + access.can_create_tag?(@pipeline.ref) + else + true # Allow it for now and we'll reject when we check ref existence + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb new file mode 100644 index 00000000000..489bcd79655 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -0,0 +1,35 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Config < Chain::Base + include Chain::Helpers + + def perform! + unless @pipeline.config_processor + unless @pipeline.ci_yaml_file + return error("Missing #{@pipeline.ci_yaml_file_path} file") + end + + if @command.save_incompleted && @pipeline.has_yaml_errors? + @pipeline.drop + end + + return error(@pipeline.yaml_errors) + end + + unless @pipeline.has_stage_seeds? + return error('No stages / jobs for this pipeline.') + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb new file mode 100644 index 00000000000..70a4cfdbdea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class Repository < Chain::Base + include Chain::Helpers + + def perform! + unless branch_exists? || tag_exists? + return error('Reference not found') + end + + ## TODO, we check commit in the service, that is why + # there is no repository access here. + # + unless pipeline.sha + return error('Commit not found') + end + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb new file mode 100644 index 00000000000..469fc094cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -0,0 +1,143 @@ +module Gitlab + module Ci + module Pipeline + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module Duration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb deleted file mode 100644 index 3208cc2bef6..00000000000 --- a/lib/gitlab/ci/pipeline_duration.rb +++ /dev/null @@ -1,141 +0,0 @@ -module Gitlab - module Ci - # # Introduction - total running time - # - # The problem this module is trying to solve is finding the total running - # time amongst all the jobs, excluding retries and pending (queue) time. - # We could reduce this problem down to finding the union of periods. - # - # So each job would be represented as a `Period`, which consists of - # `Period#first` as when the job started and `Period#last` as when the - # job was finished. A simple example here would be: - # - # * A (1, 3) - # * B (2, 4) - # * C (6, 7) - # - # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. - # C begins from 6, and ends to 7. Visually it could be viewed as: - # - # 0 1 2 3 4 5 6 7 - # AAAAAAA - # BBBBBBB - # CCCC - # - # The union of A, B, and C would be (1, 4) and (6, 7), therefore the - # total running time should be: - # - # (4 - 1) + (7 - 6) => 4 - # - # # The Algorithm - # - # The algorithm used here for union would be described as follow. - # First we make sure that all periods are sorted by `Period#first`. - # Then we try to merge periods by iterating through the first period - # to the last period. The goal would be merging all overlapped periods - # so that in the end all the periods are discrete. When all periods - # are discrete, we're free to just sum all the periods to get real - # running time. - # - # Here we begin from A, and compare it to B. We could find that - # before A ends, B already started. That is `B.first <= A.last` - # that is `2 <= 3` which means A and B are overlapping! - # - # When we found that two periods are overlapping, we would need to merge - # them into a new period and disregard the old periods. To make a new - # period, we take `A.first` as the new first because remember? we sorted - # them, so `A.first` must be smaller or equal to `B.first`. And we take - # `[A.last, B.last].max` as the new last because we want whoever ended - # later. This could be broken into two cases: - # - # 0 1 2 3 4 - # AAAAAAA - # BBBBBBB - # - # Or: - # - # 0 1 2 3 4 - # AAAAAAAAAA - # BBBB - # - # So that we need to take whoever ends later. Back to our example, - # after merging and discard A and B it could be visually viewed as: - # - # 0 1 2 3 4 5 6 7 - # DDDDDDDDDD - # CCCC - # - # Now we could go on and compare the newly created D and the old C. - # We could figure out that D and C are not overlapping by checking - # `C.first <= D.last` is `false`. Therefore we need to keep both C - # and D. The example would end here because there are no more jobs. - # - # After having the union of all periods, we just need to sum the length - # of all periods to get total time. - # - # (4 - 1) + (7 - 6) => 4 - # - # That is 4 is the answer in the example. - module PipelineDuration - extend self - - Period = Struct.new(:first, :last) do - def duration - last - first - end - end - - def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.builds.latest - .where(status: status).where.not(started_at: nil).order(:started_at) - - from_builds(builds) - end - - def from_builds(builds) - now = Time.now - - periods = builds.map do |b| - Period.new(b.started_at, b.finished_at || now) - end - - from_periods(periods) - end - - # periods should be sorted by `first` - def from_periods(periods) - process_duration(process_periods(periods)) - end - - private - - def process_periods(periods) - return periods if periods.empty? - - periods.drop(1).inject([periods.first]) do |result, current| - previous = result.last - - if overlap?(previous, current) - result[-1] = merge(previous, current) - result - else - result << current - end - end - end - - def overlap?(previous, current) - current.first <= previous.last - end - - def merge(previous, current) - Period.new(previous.first, [previous.last, current.last].max) - end - - def process_duration(periods) - periods.sum(&:duration) - end - end - end -end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 7582964b24e..0bd78b03448 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -5,12 +5,11 @@ module Gitlab include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - attr_reader :path, :cache, :stages, :jobs + attr_reader :cache, :stages, :jobs - def initialize(config, path = nil) + def initialize(config) @ci_config = Gitlab::Ci::Config.new(config) @config = @ci_config.to_hash - @path = path unless @ci_config.valid? raise ValidationError, @ci_config.errors.first @@ -21,28 +20,12 @@ module Gitlab raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds @jobs.map do |name, _| build_attributes(name) end end - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) - - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end - - seeds.compact - end - def build_attributes(name) job = @jobs[name.to_sym] || {} @@ -70,6 +53,32 @@ module Gitlab }.compact } end + def pipeline_stage_builds(stage, pipeline) + selected_jobs = @jobs.select do |_, job| + next unless job[:stage] == stage + + only_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:only, {})) + except_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:except, {})) + + only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline) } + end + + selected_jobs.map { |_, job| build_attributes(job[:name]) } + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + def self.validation_message(content) return 'Please provide content of .gitlab-ci.yml' if content.blank? @@ -83,34 +92,6 @@ module Gitlab private - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_active? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - def initial_parsing ## # Global config @@ -203,51 +184,6 @@ module Gitlab raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" end end - - def process?(only_params, except_params, ref, tag, source) - if only_params.present? - return false unless matching?(only_params, ref, tag, source) - end - - if except_params.present? - return false if matching?(except_params, ref, tag, source) - end - - true - end - - def matching?(patterns, ref, tag, source) - patterns.any? do |pattern| - pattern, path = pattern.split('@', 2) - matches_path?(path) && matches_pattern?(pattern, ref, tag, source) - end - end - - def matches_path?(path) - return true unless path - - path == self.path - end - - def matches_pattern?(pattern, ref, tag, source) - return true if tag && pattern == 'tags' - return true if !tag && pattern == 'branches' - return true if source_to_pattern(source) == pattern - - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end end end end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 58f86abc5c4..243c1f1394d 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,7 +1,7 @@ module Gitlab class ClosingIssueExtractor ISSUE_CLOSING_REGEX = begin - link_pattern = URI.regexp(%w(http https)) + link_pattern = Banzai::Filter::AutolinkFilter::LINK_PATTERN pattern = Gitlab.config.gitlab.issue_closing_pattern pattern = pattern.sub('%{issue_ref}', "(?:(?:#{link_pattern})|(?:#{Issue.reference_pattern}))") diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 4ab5b3455a5..31a46a738c3 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -64,8 +64,11 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. - commit_attrs = commits_limited.map do |commit| - commit.hook_attrs(with_changed_files: true) + # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 + commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do + commits_limited.map do |commit| + commit.hook_attrs(with_changed_files: true) + end end type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 371cbe04b9b..c98eefbce25 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - base_sha == other.base_sha && - start_sha == other.start_sha && - head_sha == other.head_sha + shas_equal?(base_sha, other.base_sha) && + shas_equal?(start_sha, other.start_sha) && + shas_equal?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,6 +47,22 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end + + private + + def shas_equal?(sha1, sha2) + return true if sha1 == sha2 + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + length = [sha1.length, sha2.length].min + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + return false if length < Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 1dabd4ebdd0..fcac85ff892 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -5,7 +5,7 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false # Finding a viewer for a diff file happens based only on extension and whether the # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, @@ -166,7 +166,7 @@ module Gitlab end def binary? - old_blob&.binary? || new_blob&.binary? + has_binary_notice? || old_blob&.binary? || new_blob&.binary? end def text? diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f80afb20f0c..b8db3adef0a 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -49,12 +49,13 @@ module Gitlab coder['attributes'] = self.to_h end - def key - @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] - end - def ==(other) - other.is_a?(self.class) && key == other.key + other.is_a?(self.class) && + other.diff_refs == diff_refs && + other.old_path == old_path && + other.new_path == new_path && + other.old_line == old_line && + other.new_line == new_line end def to_h diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index abd401224d8..c5a8ea12245 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -284,13 +284,18 @@ module Gitlab EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch was found in the EE repository. + If you're a community contributor, don't worry, someone from + GitLab Inc. will take care of this, and you don't have to do anything. + If you're willing to help, and are ok to contribute to EE as well, + you're welcome to help. You could follow the instructions below. + #{conflicting_files_msg} We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch that includes changes from `#{ce_branch}` but also specific changes than can be applied cleanly to EE/master. In some cases, the conflicts are trivial and you can ignore the warning from this job. As always, - use your best judgment! + use your best judgement! There are different ways to create such branch: diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index b984492d369..455814a9159 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -29,6 +29,8 @@ module Gitlab # http://gitlab.com/some/link/#1234, and code `puts #1234`' # class ReferenceRewriter + RewriteError = Class.new(StandardError) + def initialize(text, source_project, current_user) @text = text @source_project = source_project @@ -61,6 +63,10 @@ module Gitlab cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference + if cross_reference.nil? + raise RewriteError, "Unspecified reference detected for #{referable.class.name}" + end + new_text = before + cross_reference + after substitution_valid?(new_text) ? cross_reference : reference end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index b4b6326cfdd..c78fe63f9b5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -57,6 +57,15 @@ module Gitlab def version Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) end + + def check_namespace!(*objects) + expected_namespace = self.name + '::' + objects.each do |object| + unless object.class.name.start_with?(expected_namespace) + raise ArgumentError, "expected object in #{expected_namespace}, got #{object}" + end + end + end end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 8d96826f6ee..a4336facee5 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -32,6 +32,8 @@ module Gitlab else blob = repository.lookup(sha) + next unless blob.is_a?(Rugged::Blob) + new( id: blob.oid, size: blob.size, diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 1f370686186..1957c254c28 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -413,6 +413,10 @@ module Gitlab end end + def merge_commit? + parent_ids.size > 1 + end + private def init_from_hash(hash) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a23c8cf0dd1..ca94b4baa59 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -24,41 +24,13 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze - class << self - # The maximum size of a diff to display. - def size_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit - else - find_size_limit - end - end - - # The maximum size before a diff is collapsed. - def collapse_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit - else - find_collapse_limit - end - end - - def find_size_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 200.kilobytes - else - 100.kilobytes - end - end + # The maximum size of a diff to display. + SIZE_LIMIT = 100.kilobytes - def find_collapse_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 100.kilobytes - else - 10.kilobytes - end - end + # The maximum size before a diff is collapsed. + COLLAPSE_LIMIT = 10.kilobytes + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -172,7 +144,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.size_limit + @too_large = @diff.bytesize >= SIZE_LIMIT else @too_large end @@ -190,7 +162,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end def collapse! @@ -206,6 +178,10 @@ module Gitlab Diff.binary_message(@old_path, @new_path) end + def has_binary_notice? + @diff.start_with?('Binary') + end + private def init_from_rugged(rugged) @@ -271,14 +247,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= self.class.size_limit + if size >= SIZE_LIMIT too_large! return true end end end - if !expanded && size >= self.class.collapse_limit + if !expanded && size >= COLLAPSE_LIMIT collapse! return true end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index cc35d77c6e4..e29a1f7afa1 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -22,22 +22,22 @@ module Gitlab File.exist?(path) end - def trigger(gl_id, oldrev, newrev, ref) + def trigger(gl_id, gl_username, oldrev, newrev, ref) return [true, nil] unless exists? Bundler.with_clean_env do case name when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) + call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + call_update_hook(gl_id, gl_username, oldrev, newrev, ref) end end end private - def call_receive_hook(gl_id, oldrev, newrev, ref) + def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") exit_status = false @@ -45,6 +45,7 @@ module Gitlab vars = { 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, 'GL_REPOSITORY' => repository.gl_repository @@ -80,16 +81,21 @@ module Gitlab [exit_status, exit_message] end - def call_update_hook(gl_id, oldrev, newrev, ref) + def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) Dir.chdir(repo_path) do - stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) - [status.success?, stderr.presence || stdout] + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username + } + stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end end def retrieve_error_message(stderr, stdout) - err_message = stderr.gets - err_message.blank? ? stdout.gets : err_message + err_message = stderr.read + err_message = err_message.blank? ? stdout.read : err_message + err_message.gsub(/\R/, "<br>").html_safe end end end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb index ea8a87a1290..c327e9b1616 100644 --- a/lib/gitlab/git/hooks_service.rb +++ b/lib/gitlab/git/hooks_service.rb @@ -5,12 +5,13 @@ module Gitlab attr_accessor :oldrev, :newrev, :ref - def execute(committer, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = committer.gl_id - @oldrev = oldrev - @newrev = newrev - @ref = ref + def execute(pusher, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = pusher.gl_id + @gl_username = pusher.name + @oldrev = oldrev + @newrev = newrev + @ref = ref %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) @@ -29,7 +30,7 @@ module Gitlab def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, oldrev, newrev, ref) + hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index dcdec818f5e..d835dcca8ba 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,6 +1,8 @@ module Gitlab module Git class OperationService + include Gitlab::Git::Popen + WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created @@ -15,9 +17,7 @@ module Gitlab end # Refactoring aid - unless new_repository.is_a?(Gitlab::Git::Repository) - raise "expected a Gitlab::Git::Repository, got #{new_repository}" - end + Gitlab::Git.check_namespace!(new_repository) @repository = new_repository end @@ -152,13 +152,15 @@ module Gitlab # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - _, status = Gitlab::Popen.popen( + + output, status = popen( command, repository.path) do |stdin| stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") end unless status.zero? + Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") raise Gitlab::Git::CommitError.new( "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ " Please refresh and try again.") diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 25fa62ce4bd..3d2fc471d28 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -5,17 +5,21 @@ require 'open3' module Gitlab module Git module Popen - def popen(cmd, path) + def popen(cmd, path, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end - vars = { "PWD" => path } + path ||= Dir.pwd + vars['PWD'] = path options = { chdir: path } @cmd_output = "" @cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + yield(stdin) if block_given? + stdin.close + @cmd_output << stdout.read @cmd_output << stderr.read @cmd_status = wait_thr.value.exitstatus diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 18210bcab4e..89b654253cb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -19,13 +19,12 @@ module Gitlab InvalidRef = Class.new(StandardError) GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) + CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the storage path, not the storage name - def create(storage_path, name, bare: true, symlink_hooks_to: nil) - repo_path = File.join(storage_path, name) - repo_path += '.git' unless repo_path.end_with?('.git') - + # Unlike `new`, `create` takes the repository path + def create(repo_path, bare: true, symlink_hooks_to: nil) FileUtils.mkdir_p(repo_path, mode: 0770) # Equivalent to `git --git-path=#{repo_path} init [--bare]` @@ -54,14 +53,15 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver - # 'path' must be the path to a _bare_ git repository, e.g. - # /path/to/my-repo.git + # This initializer method is only used on the client side (gitlab-ce). + # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -72,8 +72,6 @@ module Gitlab delegate :empty?, to: :rugged - delegate :exists?, to: :gitaly_repository_client - def ==(other) path == other.path end @@ -101,6 +99,18 @@ module Gitlab @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end + def exists? + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + gitaly_repository_client.exists? + else + circuit_breaker.perform do + File.exist?(File.join(@path, 'refs')) + end + end + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -180,6 +190,28 @@ module Gitlab end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -385,7 +417,13 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(self, c) } + gitaly_migrate(:find_commits) do |is_enabled| + if is_enabled + gitaly_commit_client.find_commits(options) + else + raw_log(options).map { |c| Commit.decorate(self, c) } + end + end end # Used in gitaly-ruby @@ -474,7 +512,15 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + iterator = gitaly_migrate(:diff_between) do |is_enabled| + if is_enabled + gitaly_commit_client.diff(from, to, options.merge(paths: paths)) + else + diff_patches(from, to, options, *paths) + end + end + + Gitlab::Git::DiffCollection.new(iterator, options) end # Returns a RefName for a given SHA @@ -489,7 +535,7 @@ module Gitlab # Not found -> ["", 0] # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - Gitlab::Popen.popen(args, @path).first.split.last + popen(args, @path).first.split.last end end end @@ -611,42 +657,43 @@ module Gitlab end def add_branch(branch_name, user:, target:) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + gitaly_migrate(:operation_user_create_branch) do |is_enabled| + if is_enabled + gitaly_add_branch(branch_name, user, target) + else + rugged_add_branch(branch_name, user, target) + end + end end def add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } + gitaly_migrate(:operation_user_add_tag) do |is_enabled| + if is_enabled + gitaly_add_tag(tag_name, user: user, target: target, message: message) + else + rugged_add_tag(tag_name, user: user, target: target, message: message) + end end - - OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex end def rm_branch(branch_name, user:) - OperationService.new(user, self).rm_branch(find_branch(branch_name)) + gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_delete_branch(branch_name, user) + else + OperationService.new(user, self).rm_branch(find_branch(branch_name)) + end + end end def rm_tag(tag_name, user:) - OperationService.new(user, self).rm_tag(find_tag(tag_name)) + gitaly_migrate(:operation_user_delete_tag) do |is_enabled| + if is_enabled + gitaly_operations_client.rm_tag(tag_name, user) + else + Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) + end + end end def find_tag(name) @@ -684,6 +731,88 @@ module Gitlab nil end + def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + revert_tree_id = check_revert_content(commit, start_commit.sha) + raise CreateTreeError unless revert_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) + end + end + + def check_revert_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << { mainline: 1 } if target_commit.merge_commit? + + revert_index = rugged.revert_commit(*args) + return false if revert_index.conflicts? + + tree_id = revert_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + + def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) + raise CreateTreeError unless cherry_pick_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) + end + end + + def check_cherry_pick_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << 1 if target_commit.merge_commit? + + cherry_pick_index = rugged.cherrypick_commit(*args) + return false if cherry_pick_index.conflicts? + + tree_id = cherry_pick_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + + def diff_exists?(sha1, sha2) + rugged.diff(sha1, sha2).size > 0 + end + + def user_to_committer(user) + Gitlab::Git.committer_hash(email: user.email, name: user.name) + end + def create_commit(params = {}) params[:message].delete!("\r") @@ -709,9 +838,7 @@ module Gitlab end command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - message, status = Gitlab::Popen.popen( - command, - path) do |stdin| + message, status = popen(command, path) do |stdin| stdin.write(instructions.join) end @@ -835,14 +962,18 @@ module Gitlab end def with_repo_branch_commit(start_repository, start_branch_name) - raise "expected Gitlab::Git::Repository, got #{start_repository}" unless start_repository.is_a?(Gitlab::Git::Repository) + Gitlab::Git.check_namespace!(start_repository) return yield nil if start_repository.empty_repo? if start_repository == self yield commit(start_branch_name) else - sha = start_repository.commit(start_branch_name).sha + start_commit = start_repository.commit(start_branch_name) + + return yield nil unless start_commit + + sha = start_commit.sha if branch_commit = commit(sha) yield branch_commit @@ -857,9 +988,9 @@ module Gitlab def with_repo_tmp_commit(start_repository, start_branch_name, sha) tmp_ref = fetch_ref( - start_repository.path, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - "refs/tmp/#{SecureRandom.hex}/head" + start_repository, + source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + target_ref: "refs/tmp/#{SecureRandom.hex}/head" ) yield commit(sha) @@ -871,8 +1002,9 @@ module Gitlab with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) + true else - raise Rugged::ReferenceError, 'source repository is empty' + false end end end @@ -890,13 +1022,27 @@ module Gitlab end end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) + def write_ref(ref_path, ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + + command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] + input = "update #{ref_path}\x00#{ref}\x00\x00" + output, status = circuit_breaker.perform do + popen(command, path) { |stdin| stdin.write(input) } + end + + raise GitError, output unless status.zero? end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - message, status = run_git(args) + def fetch_ref(source_repository, source_ref:, target_ref:) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| + if is_enabled + gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) + else + local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) + end + end # Make sure ref was created, and raise Rugged::ReferenceError when not raise Rugged::ReferenceError, message if status != 0 @@ -905,9 +1051,9 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args) + def run_git(args, env: {}) circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) + popen([Gitlab.config.git.bin_path, *args], path, env) end end @@ -931,11 +1077,17 @@ module Gitlab # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + end + + def gitaly_operations_client + @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) end def gitaly_ref_client @@ -950,6 +1102,10 @@ module Gitlab @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) end + def gitaly_operation_client + @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1272,6 +1428,33 @@ module Gitlab false end + def gitaly_add_tag(tag_name, user:, target:, message: nil) + gitaly_operations_client.add_tag(tag_name, user, target, message) + end + + def rugged_add_tag(tag_name, user:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) + } + end + + Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + rescue Rugged::TagError + raise TagExistsError + end + def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1314,6 +1497,46 @@ module Gitlab file.write(gitattributes_content) end end + + def gitaly_add_branch(branch_name, user, target) + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex + end + + def rugged_add_branch(branch_name, user, target) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(user, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def local_fetch_ref(source_path, source_ref:, target_ref:) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + run_git(args) + end + + def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_resolver.address(source_repository.storage) + gitaly_token = gitaly_resolver.token(source_repository.storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + + run_git(args, env: env) + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 2b5785a1f08..92a6a672534 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -3,6 +3,8 @@ module Gitlab module Git class RevList + include Gitlab::Git::Popen + attr_reader :oldrev, :newrev, :path_to_repo def initialize(path_to_repo:, newrev:, oldrev: nil) @@ -26,10 +28,10 @@ module Gitlab private def execute(args) - output, status = Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" end output.split("\n") diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb index e28be4b8a38..08e6c29abad 100644 --- a/lib/gitlab/git/storage.rb +++ b/lib/gitlab/git/storage.rb @@ -11,6 +11,7 @@ module Gitlab end CircuitOpen = Class.new(Inaccessible) + Misconfiguration = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 9ea9367d4b7..1eaa2d83fb6 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -28,14 +28,26 @@ module Gitlab def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| - hash[storage_name] = new(storage_name) + hash[storage_name] = build(storage_name) end end cached_circuitbreakers[storage] end - def initialize(storage, hostname = Gitlab::Environment.hostname) + def self.build(storage, hostname = Gitlab::Environment.hostname) + config = Gitlab.config.repositories.storages[storage] + + if !config.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) + elsif !config['path'].present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) + else + new(storage, hostname) + end + end + + def initialize(storage, hostname) @storage = storage @hostname = hostname @@ -64,6 +76,10 @@ module Gitlab recent_failure || too_many_failures end + def failure_info + @failure_info ||= get_failure_info + end + # Memoizing the `storage_available` call means we only do it once per # request when the storage is available. # @@ -121,10 +137,12 @@ module Gitlab end end - def failure_info - @failure_info ||= get_failure_info + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end + private + def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -134,10 +152,6 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end end end end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 2d723147f4f..1564e94b7f7 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -78,7 +78,7 @@ module Gitlab def failing_circuit_breakers @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.new(storage_name, hostname) + CircuitBreaker.build(storage_name, hostname) end end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb new file mode 100644 index 00000000000..297c043d054 --- /dev/null +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -0,0 +1,47 @@ +module Gitlab + module Git + module Storage + class NullCircuitBreaker + # These will have actual values + attr_reader :storage, + :hostname + + # These will always have nil values + attr_reader :storage_path, + :failure_wait_time, + :failure_reset_time, + :storage_timeout + + def initialize(storage, hostname, error: nil) + @storage = storage + @hostname = hostname + @error = error + end + + def perform + @error ? raise(@error) : yield + end + + def circuit_broken? + !!@error + end + + def failure_count_threshold + 1 + end + + def last_failure + circuit_broken? ? Time.now : nil + end + + def failure_count + circuit_broken? ? 1 : 0 + end + + def failure_info + Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + end + end + end + end +end diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index ea634d39668..da74719ae87 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -1,24 +1,26 @@ module Gitlab module Git class User - attr_reader :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) end + # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628 def self.from_gitaly(gitaly_user) - new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end - def initialize(name, email, gl_id) + def initialize(username, name, email, gl_id) + @username = username @name = name @email = email @gl_id = gl_id end def ==(other) - [name, email, gl_id] == [other.name, other.email, other.gl_id] + [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb new file mode 100644 index 00000000000..d651c931a38 --- /dev/null +++ b/lib/gitlab/git/wiki.rb @@ -0,0 +1,115 @@ +module Gitlab + module Git + class Wiki + DuplicatePageError = Class.new(StandardError) + + CommitDetails = Struct.new(:name, :email, :message) do + def to_h + { name: name, email: email, message: message } + end + end + + def self.default_ref + 'master' + end + + # Initialize with a Gitlab::Git::Repository instance + def initialize(repository) + @repository = repository + end + + def repository_exists? + @repository.exists? + end + + def write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def delete_page(page_path, commit_details) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) + nil + end + + def update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + + def pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + + def page(title:, version: nil, dir: nil) + if version + version = Gitlab::Git::Commit.find(@repository, version).id + end + + gollum_page = gollum_wiki.page(title, version, dir) + return unless gollum_page + + new_page(gollum_page) + end + + def file(name, version) + version ||= self.class.default_ref + gollum_file = gollum_wiki.file(name, version) + return unless gollum_file + + Gitlab::Git::WikiFile.new(gollum_file) + end + + def page_versions(page_path) + current_page = gollum_page_by_path(page_path) + current_page.versions.map do |gollum_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) + new_version(gollum_page, gollum_git_commit.id) + end + end + + def preview_slug(title, format) + gollum_wiki.preview_page(title, '', format).url_path + end + + private + + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + + def gollum_page_by_path(page_path) + page_name = Gollum::Page.canonicalize_filename(page_path) + page_dir = File.split(page_path).first + + gollum_wiki.paged(page_name, page_dir) + end + + def new_page(gollum_page) + Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) + end + + def new_version(gollum_page, commit_id) + commit = Gitlab::Git::Commit.find(@repository, commit_id) + Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + end + + def assert_type!(object, klass) + unless object.is_a?(klass) + raise ArgumentError, "expected a #{klass}, got #{object.inspect}" + end + end + end + end +end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb new file mode 100644 index 00000000000..527f2a44dea --- /dev/null +++ b/lib/gitlab/git/wiki_file.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiFile + attr_reader :mime_type, :raw_data, :name + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::File is not serializable we must get all the data from + # 'gollum_file' during initialization, and NOT store it in an instance + # variable. + def initialize(gollum_file) + @mime_type = gollum_file.mime_type + @raw_data = gollum_file.raw_data + @name = gollum_file.name + end + end + end +end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb new file mode 100644 index 00000000000..a06bac4414f --- /dev/null +++ b/lib/gitlab/git/wiki_page.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + class WikiPage + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::Page is not serializable we must get all the data from + # 'gollum_page' during initialization, and NOT store it in an instance + # variable. + # + # Note that 'version' is a WikiPageVersion instance which it itself + # serializable. That means it's OK to store 'version' in an instance + # variable. + def initialize(gollum_page, version) + @url_path = gollum_page.url_path + @title = gollum_page.title + @format = gollum_page.format + @path = gollum_page.path + @raw_data = gollum_page.raw_data + @name = gollum_page.name + @historical = gollum_page.historical? + + @version = version + end + + def historical? + @historical + end + + def text_data + return @text_data if defined?(@text_data) + + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end + end + end +end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb new file mode 100644 index 00000000000..55f1afedcab --- /dev/null +++ b/lib/gitlab/git/wiki_page_version.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiPageVersion + attr_reader :commit, :format + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are + # serializable. + def initialize(commit, format) + @commit = commit + @format = format + end + + delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 62d1ecae676..db67ede9d9e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -16,7 +16,8 @@ module Gitlab account_blocked: 'Your account has been blocked.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', - receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', + readonly: 'The repository is temporarily read-only. Please try again later.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze @@ -159,6 +160,10 @@ module Gitlab end def check_push_access!(changes) + if project.repository_read_only? + raise UnauthorizedError, ERROR_MESSAGES[:readonly] + end + if deploy_key check_deploy_key_push_access! elsif user diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cbd9ff406de..87b300dcf7e 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -151,7 +151,7 @@ module Gitlab actual_call_count = increment_call_count("gitaly_#{call_site}_actual") # Do no enforce limits in production - return if Rails.env.production? + return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"] # Check if this call is nested within a allow_n_plus_1_calls # block and skip check if it is @@ -228,10 +228,20 @@ module Gitlab path.read.chomp end + def self.timestamp(t) + Google::Protobuf::Timestamp.new(seconds: t.to_i) + end + def self.encode(s) + return "" if s.nil? + s.dup.force_encoding(Encoding::ASCII_8BIT) end + def self.encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) + end + # Count a stack. Used for n+1 detection def self.count_stack return unless RequestStore.active? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ba1a7830a4..a2b50f2507e 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -32,20 +32,38 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value end + def diff(from, to, options = {}) + from_id = case from + when NilClass + EMPTY_TREE_ID + when Rugged::Commit + from.oid + else + from + end + + to_id = case to + when NilClass + EMPTY_TREE_ID + when Rugged::Commit + to.oid + else + to + end + + request_params = diff_between_commits_request_params(from_id, to_id, options) + + call_commit_diff(request_params, options) + end + def diff_from_parent(commit, options = {}) - request_params = commit_diff_request_params(commit, options) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - request_params[:enforce_limits] = options.fetch(:limits, true) - request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params = diff_from_parent_request_params(commit, options) - request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) - GitalyClient::DiffStitcher.new(response) + call_commit_diff(request_params, options) end def commit_deltas(commit) - request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) + request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) response.flat_map { |msg| msg.deltas } @@ -212,16 +230,51 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request) end + def find_commits(options) + request = Gitaly::FindCommitsRequest.new( + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + disable_walk: options[:disable_walk] + ) + request.after = GitalyClient.timestamp(options[:after]) if options[:after] + request.before = GitalyClient.timestamp(options[:before]) if options[:before] + request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + + request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request) + + consume_commits_response(response) + end + private - def commit_diff_request_params(commit, options = {}) + def call_commit_diff(request_params, options = {}) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + request_params[:enforce_limits] = options.fetch(:limits, true) + request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + + request = Gitaly::CommitDiffRequest.new(request_params) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + GitalyClient::DiffStitcher.new(response) + end + + def diff_from_parent_request_params(commit, options = {}) parent_id = commit.parent_ids.first || EMPTY_TREE_ID + diff_between_commits_request_params(parent_id, commit.id, options) + end + + def diff_between_commits_request_params(from_id, to_id, options) { repository: @gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - paths: options.fetch(:paths, []) + left_commit_id: from_id, + right_commit_id: to_id, + paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } } end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb new file mode 100644 index 00000000000..81ddaf13e10 --- /dev/null +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -0,0 +1,79 @@ +module Gitlab + module GitalyClient + class OperationService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def rm_tag(tag_name, user) + request = Gitaly::UserDeleteTagRequest.new( + repository: @gitaly_repo, + tag_name: GitalyClient.encode(tag_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + + def add_tag(tag_name, user, target, message) + request = Gitaly::UserCreateTagRequest.new( + repository: @gitaly_repo, + user: Util.gitaly_user(user), + tag_name: GitalyClient.encode(tag_name), + target_revision: GitalyClient.encode(target), + message: GitalyClient.encode(message.to_s) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + elsif response.exists + raise Gitlab::Git::Repository::TagExistsError + end + + Util.gitlab_tag_from_gitaly_tag(@repository, response.tag) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Repository::InvalidRef, e + end + + def user_create_branch(branch_name, user, start_point) + request = Gitaly::UserCreateBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user), + start_point: GitalyClient.encode(start_point) + ) + response = GitalyClient.call(@repository.storage, :operation_service, + :user_create_branch, request) + if response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + end + + branch = response.branch + return nil unless branch + + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + end + + def user_delete_branch(branch_name, user) + request = Gitaly::UserDeleteBranchRequest.new( + repository: @gitaly_repo, + branch_name: GitalyClient.encode(branch_name), + user: Util.gitaly_user(user) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8ef873d5848..8214b7d63fa 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -57,6 +57,14 @@ module Gitlab branch_names.count end + # TODO implement a more efficient RPC for this https://gitlab.com/gitlab-org/gitaly/issues/616 + def has_local_branches? + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request).first + + response&.names.present? + end + def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by @@ -155,19 +163,7 @@ module Gitlab def consume_tags_response(response) response.flat_map do |message| - message.tags.map do |gitaly_tag| - if gitaly_tag.target_commit.present? - gitaly_commit = Gitlab::Git::Commit.decorate(@repository, gitaly_tag.target_commit) - end - - Gitlab::Git::Tag.new( - @repository, - encode!(gitaly_tag.name.dup), - gitaly_tag.id, - gitaly_commit, - encode!(gitaly_tag.message.chomp) - ) - end + message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 177a1284f38..fdf912214e0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -53,6 +53,11 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :fetch_remote, request) end + + def create_repository + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :create_repository, request) + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 8fc937496af..da43c616b94 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -2,14 +2,39 @@ module Gitlab module GitalyClient module Util class << self - def repository(repository_storage, relative_path) + def repository(repository_storage, relative_path, gl_repository) Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, + gl_repository: gl_repository, git_object_directory: Gitlab::Git::Env['GIT_OBJECT_DIRECTORY'].to_s, git_alternate_object_directories: Array.wrap(Gitlab::Git::Env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) ) end + + def gitaly_user(gitlab_user) + return unless gitlab_user + + Gitaly::User.new( + gl_id: Gitlab::GlId.gl_id(gitlab_user), + name: GitalyClient.encode(gitlab_user.name), + email: GitalyClient.encode(gitlab_user.email) + ) + end + + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) + if gitaly_tag.target_commit.present? + commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) + end + + Gitlab::Git::Tag.new( + repository, + Gitlab::EncodingHelper.encode!(gitaly_tag.name.dup), + gitaly_tag.id, + commit, + Gitlab::EncodingHelper.encode!(gitaly_tag.message.chomp) + ) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9bcc579278f..3a666c2268b 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,6 +20,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') + gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index a533d4364ef..afaa59b1018 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -58,11 +58,11 @@ module Gitlab end def repository_storages - @repository_storage ||= storages_paths.keys + storages_paths.keys end def storages_paths - @storage_paths ||= Gitlab.config.repositories.storages + Gitlab.config.repositories.storages end def exec_with_timeout(cmd_args, *args, &block) @@ -125,7 +125,7 @@ module Gitlab end def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" } + Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } rescue Gitlab::Git::Storage::Inaccessible nil end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index cdbdfa10d0e..da43bd0af4b 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -113,7 +113,7 @@ module Gitlab def kubeconfig_embed_ca_pem(config, ca_pem) cluster = config.dig(:clusters, 0, :cluster) - cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + cluster[:'certificate-authority-data'] = Base64.strict_encode64(ca_pem) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index fb68627dedf..e60ceba27c8 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,7 +16,7 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute true else diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index cd7e4ca7b7e..0afaa2306b5 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -22,8 +22,8 @@ module Gitlab Gitlab::LDAP::Config.new(provider) end - def users(field, value, limit = nil) - options = user_options(field, value, limit) + def users(fields, value, limit = nil) + options = user_options(Array(fields), value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -72,20 +72,24 @@ module Gitlab private - def user_options(field, value, limit) - options = { attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq } + def user_options(fields, value, limit) + options = { + attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + base: config.base + } + options[:size] = limit if limit - if field.to_sym == :dn + if fields.include?('dn') + raise ArgumentError, 'It is not currently possible to search the DN and other fields at the same time.' if fields.size > 1 + options[:base] = value options[:scope] = Net::LDAP::SearchScope_BaseObject - options[:filter] = user_filter else - options[:base] = config.base - options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + filter = fields.map { |field| Net::LDAP::Filter.eq(field, value) }.inject(:|) end - options + options.merge(filter: user_filter(filter)) end def user_filter(filter = nil) diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 4d6f8ac79de..9a6f7827b16 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -17,6 +17,12 @@ module Gitlab adapter.user('dn', dn) end + def self.find_by_email(email, adapter) + email_fields = adapter.config.attributes['email'] + + adapter.user(email_fields, email) + end + def self.disabled_via_active_directory?(dn, adapter) adapter.dn_matches_filter?(dn, AD_USER_DISABLED) end diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 3bf27b37ae6..1793097363e 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -17,41 +17,19 @@ module Gitlab end end - def initialize(auth_hash) - super - update_user_attributes - end - def save super('LDAP') end # instance methods - def gl_user - @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user + def find_user + find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end - def find_by_email - ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_attribute?(:email) - end - - def update_user_attributes - if persisted? - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) - - # For a new identity set extern_uid to the LDAP DN - # For an existing identity with matching email but changed DN, update the DN. - # For an existing identity with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid - end - end - def changed? gl_user.changed? || gl_user.identities.any?(&:changed?) end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb deleted file mode 100644 index 306923902e0..00000000000 --- a/lib/gitlab/markdown/pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Gitlab - module Markdown - class Pipeline - def self.[](name) - name ||= :full - const_get("#{name.to_s.camelize}Pipeline") - end - - def self.filters - [] - end - - def self.transform_context(context) - context - end - - def self.html_pipeline - @html_pipeline ||= HTML::Pipeline.new(filters) - end - - class << self - %i(call to_document to_html).each do |meth| - define_method(meth) do |text, context| - context = transform_context(context) - - html_pipeline.__send__(meth, text, context) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7704bf715e4..68815be4d13 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -13,6 +13,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash update_profile if sync_profile_from_provider? + add_or_update_user_identities end def persisted? @@ -32,7 +33,7 @@ module Gitlab block_after_save = needs_blocking? - Users::UpdateService.new(gl_user).execute! + Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block if block_after_save @@ -44,47 +45,54 @@ module Gitlab end def gl_user - @user ||= find_by_uid_and_provider + return @gl_user if defined?(@gl_user) - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end + @gl_user = find_user + end - if signup_enabled? - @user ||= build_new_user - end + def find_user + user = find_by_uid_and_provider - if external_provider? && @user - @user.external = true - end + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? + + user.external = true if external_provider? && user - @user + user end protected - def find_or_create_ldap_user + def add_or_update_user_identities + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + + identity ||= gl_user.identities.build(provider: auth_hash.provider) + identity.extern_uid = auth_hash.uid + + if auto_link_ldap_user? && !gl_user.ldap_user? && ldap_person + log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." + gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) + end + end + + def find_or_build_ldap_user return unless ldap_person - # If a corresponding person exists with same uid in a LDAP server, - # check if the user already has a GitLab account. user = Gitlab::LDAP::User.find_by_uid_and_provider(ldap_person.dn, ldap_person.provider) if user - # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account. log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity." - user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider) - else - log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account." - user = find_by_uid_and_provider - if user.nil? - log.info "No user found using #{auth_hash.provider} provider. Creating a new one." - user = build_new_user - end - log.info "Correct account has been found. Adding LDAP identity to user: #{user.username}." - user.identities.new(provider: ldap_person.provider, extern_uid: ldap_person.dn) + return user end - user + log.info "No user found using #{auth_hash.provider} provider. Creating a new one." + build_new_user + end + + def find_by_email + return unless auth_hash.has_attribute?(:email) + + ::User.find_by(email: auth_hash.email.downcase) end def auto_link_ldap_user? @@ -108,9 +116,9 @@ module Gitlab end def find_ldap_person(auth_hash, adapter) - by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_email(auth_hash.uid, adapter) || + Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) end def ldap_config @@ -152,7 +160,7 @@ module Gitlab end def build_new_user - user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) + user_params = user_attributes.merge(skip_confirmation: true) Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 0f323a9e8b2..e0a9d1dee77 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -10,41 +10,20 @@ module Gitlab super('SAML') end - def gl_user - if auto_link_ldap_user? - @user ||= find_or_create_ldap_user - end - - @user ||= find_by_uid_and_provider - - if auto_link_saml_user? - @user ||= find_by_email - end + def find_user + user = find_by_uid_and_provider - if signup_enabled? - @user ||= build_new_user - end + user ||= find_by_email if auto_link_saml_user? + user ||= find_or_build_ldap_user if auto_link_ldap_user? + user ||= build_new_user if signup_enabled? - if external_users_enabled? && @user + if external_users_enabled? && user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - @user.external = - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - false - else - true - end + user.external = !(auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? end - @user - end - - def find_by_email - if auth_hash.has_attribute?(:email) - user = ::User.find_by(email: auth_hash.email.downcase) - user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user - user - end + user end def changed? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 81ecdf43ef9..a99f8e2b5f8 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -65,7 +65,7 @@ module Gitlab # Init new repository # - # storage - project's storage path + # storage - project's storage name # name - project path with namespace # # Ex. @@ -73,7 +73,19 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Git::Repository.create(storage, name, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + relative_path = name.dup + relative_path << '.git' unless relative_path.end_with?('.git') + + gitaly_migrate(:create_repository) do |is_enabled| + if is_enabled + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + repository.gitaly_repository_client.create_repository + true + else + repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) + end + end rescue => err Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 104280f520a..d7d24eeb37b 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,7 +25,7 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']} "\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 222021e8802..f30c771837a 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -12,8 +12,9 @@ module Gitlab # # Project.where("id IN (#{sql})") class Union - def initialize(relations) + def initialize(relations, remove_duplicates: true) @relations = relations + @remove_duplicates = remove_duplicates end def to_sql @@ -25,7 +26,11 @@ module Gitlab @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end - fragments.join("\nUNION\n") + fragments.join("\n#{union_keyword}\n") + end + + def union_keyword + @remove_duplicates ? 'UNION' : 'UNION ALL' end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 4e1ec1402ea..1caa791c1be 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,9 @@ module Gitlab class UrlSanitizer + ALLOWED_SCHEMES = %w[http https ssh git].freeze + def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) + regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -11,9 +13,9 @@ module Gitlab def self.valid?(url) return false unless url.present? - Addressable::URI.parse(url.strip) + uri = Addressable::URI.parse(url.strip) - true + ALLOWED_SCHEMES.include?(uri.scheme) rescue Addressable::URI::InvalidURIError false end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 36708078136..6857038dba8 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,12 +9,28 @@ module Gitlab def uncached_data license_usage_data.merge(system_usage_data) + .merge(features_usage_data) + .merge(components_usage_data) end def to_json(force_refresh: false) data(force_refresh: force_refresh).to_json end + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + def system_usage_data { counts: { @@ -54,18 +70,28 @@ module Gitlab } end - def license_usage_data - usage_data = { - uuid: current_application_settings.uuid, - hostname: Gitlab.config.gitlab.host, - version: Gitlab::VERSION, - active_user_count: User.active.count, - recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, - edition: 'CE' + def features_usage_data + features_usage_data_ce + end + + def features_usage_data_ce + { + signup: current_application_settings.signup_enabled?, + ldap: Gitlab.config.ldap.enabled, + gravatar: current_application_settings.gravatar_enabled?, + omniauth: Gitlab.config.omniauth.enabled, + reply_by_email: Gitlab::IncomingEmail.enabled?, + container_registry: Gitlab.config.registry.enabled, + gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled } + end - usage_data + def components_usage_data + { + gitlab_pages: { enabled: Gitlab.config.pages.enabled, version: Gitlab::Pages::VERSION }, + git: { version: Gitlab::Git.version }, + database: { adapter: Gitlab::Database.adapter_name, version: Gitlab::Database.version } + } end def services_usage diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 17550cf9074..f200c694562 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,9 +22,9 @@ module Gitlab params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + GL_USERNAME: user&.username, RepoPath: repo_path } - server = { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) @@ -89,6 +89,13 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + params.merge!( + 'GitalyServer' => gitaly_server_hash(repository), + 'GitalyRepository' => repository.gitaly_repository.to_h + ) + end + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 7b486d78cf0..dfa8b8b3f5b 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -5,6 +5,7 @@ module SystemCheck # whitelisted as it may change the SSH client's behaviour dramatically. WHITELIST = %w[ authorized_keys + authorized_keys.lock authorized_keys2 known_hosts ].freeze diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index c388682dfb4..6ee8c8874ec 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%W(#{Gitlab.config.git.bin_path} --version))) end def check? diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index fd82f5f8a4a..08a2c495bd4 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -9,7 +9,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version))) end def check? diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index dee108d987b..e55bea86d3f 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -4,22 +4,17 @@ module SystemCheck set_name 'IMAP server credentials are correct?' def check? - if mailbox_config - begin - imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) - imap.starttls if config[:start_tls] - imap.login(config[:email], config[:password]) - connected = true - rescue - connected = false - end + if config + try_connect_imap + else + @error = "#{mail_room_config_path} does not have mailboxes setup" + false end - - connected end def show_error try_fixing_it( + "An error occurred: #{@error.class}: #{@error.message}", 'Check that the information in config/gitlab.yml is correct' ) for_more_information( @@ -30,15 +25,31 @@ module SystemCheck private - def mailbox_config - return @config if @config + def try_connect_imap + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.starttls if config[:start_tls] + imap.login(config[:email], config[:password]) + true + rescue => error + @error = error + false + end + + def config + @config ||= load_config + end + + def mail_room_config_path + @mail_room_config_path ||= + Rails.root.join('config', 'mail_room.yml').to_s + end - config_path = Rails.root.join('config', 'mail_room.yml').to_s - erb = ERB.new(File.read(config_path)) - erb.filename = config_path + def load_config + erb = ERB.new(File.read(mail_room_config_path)) + erb.filename = mail_room_config_path config_file = YAML.load(erb.result) - @config = config_file[:mailboxes]&.first + config_file.dig(:mailboxes, 0) end end end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 259a755d724..a42f02a84fd 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -3,8 +3,8 @@ namespace :gitlab do desc 'GitLab | Assets | Compile all frontend assets' task compile: [ 'yarn:check', - 'rake:assets:precompile', 'gettext:po_to_json', + 'rake:assets:precompile', 'webpack:compile', 'fix_urls' ] diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 7ccda04a35f..3eade7bf553 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -13,7 +13,10 @@ namespace :gitlab do args end - if Gitlab::EeCompatCheck.new(opts || {}).check + if File.basename(Rails.root) == 'gitlab-ee' + puts "Skipping EE projects" + exit 0 + elsif Gitlab::EeCompatCheck.new(opts || {}).check exit 0 else exit 1 diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 42825f29e32..0e6aed32c52 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -79,7 +79,7 @@ namespace :gitlab do if File.exist?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.repository_storage_path, + if Gitlab::Shell.new.add_repository(project.repository_storage, project.disk_path) print '.' else diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake new file mode 100644 index 00000000000..e05be4a3405 --- /dev/null +++ b/lib/tasks/gitlab/storage.rake @@ -0,0 +1,85 @@ +namespace :gitlab do + namespace :storage do + desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' + task migrate_to_hashed: :environment do + legacy_projects_count = Project.with_legacy_storage.count + + if legacy_projects_count == 0 + puts 'There are no projects using legacy storage. Nothing to do!' + + next + end + + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + + project_id_batches do |start, finish| + StorageMigratorWorker.perform_async(start, finish) + + print '.' + end + + puts ' Done!' + end + + desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' + task legacy_projects: :environment do + projects_summary(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | List existing projects using Legacy Storage' + task list_legacy_projects: :environment do + projects_list(Project.with_legacy_storage) + end + + desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' + task hashed_projects: :environment do + projects_summary(Project.with_hashed_storage) + end + + desc 'Gitlab | Storage | List existing projects using Hashed Storage' + task list_hashed_projects: :environment do + projects_list(Project.with_hashed_storage) + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def project_id_batches(&block) + Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + ids = relation.pluck(:id) + + yield ids.min, ids.max + end + end + + def projects_summary(relation) + projects_count = relation.count + puts "* Found #{projects_count} projects".color(:green) + + projects_count + end + + def projects_list(relation) + projects_count = projects_summary(relation) + + projects = relation.with_route + limit = ENV.fetch('LIMIT', 500).to_i + + return unless projects_count > 0 + + puts " ! Displaying first #{limit} projects..." if projects_count > limit + + counter = 0 + projects.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |project| + counter += 1 + + puts " - #{project.full_path} (id: #{project.id})".color(:red) + + return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator + end + end + end + end +end |