diff options
Diffstat (limited to 'lib')
177 files changed, 4254 insertions, 1833 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c37e596eb9d..8094597d238 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -61,7 +61,10 @@ module API mount ::API::V3::Variables end - before { header['X-Frame-Options'] = 'SAMEORIGIN' } + before do + header['X-Frame-Options'] = 'SAMEORIGIN' + header['X-Content-Type-Options'] = 'nosniff' + end # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index b9c7d443f6c..c1c0d344917 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,6 +42,8 @@ module API # Helper Methods for Grape Endpoint module HelperMethods + include Gitlab::Utils::StrongMemoize + def find_current_user! user = find_user_from_access_token || find_user_from_warden return unless user @@ -52,9 +54,9 @@ module API end def access_token - return @access_token if defined?(@access_token) - - @access_token = find_oauth_access_token || find_personal_access_token + strong_memoize(:access_token) do + find_oauth_access_token || find_personal_access_token + end end def validate_access_token!(scopes: []) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 19152c9f395..cdef1b546a9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -29,12 +29,11 @@ module API use :pagination end get ':id/repository/branches' do - branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) + repository = user_project.repository + branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 - Gitlab::GitalyClient.allow_n_plus_1_calls do - present paginate(branches), with: Entities::Branch, project: user_project - end + present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2685dc27252..2bc4039b019 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -117,7 +117,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = user_project.notes.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: Entities::CommitNote end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 398a7906dcb..16ae99b5c6c 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -242,10 +242,7 @@ module API end expose :merged do |repo_branch, options| - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37442 - Gitlab::GitalyClient.allow_n_plus_1_calls do - options[:project].repository.merged_to_root_ref?(repo_branch.name) - end + options[:project].repository.merged_to_root_ref?(repo_branch, options[:merged_branch_names]) end expose :protected do |repo_branch, options| @@ -478,6 +475,10 @@ module API expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user], options[:project]) end + + expose :changes_count do |merge_request, _options| + merge_request.merge_request_diff.real_size + end end class MergeRequestChanges < MergeRequest @@ -1041,6 +1042,11 @@ module API expose :value end + class PagesDomainCertificateExpiration < Grape::Entity + expose :expired?, as: :expired + expose :expiration + end + class PagesDomainCertificate < Grape::Entity expose :subject expose :expired?, as: :expired @@ -1048,12 +1054,23 @@ module API expose :certificate_text end + class PagesDomainBasic < Grape::Entity + expose :domain + expose :url + expose :certificate, + as: :certificate_expiration, + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificateExpiration do |pages_domain| + pages_domain + end + end + class PagesDomain < Grape::Entity expose :domain expose :url expose :certificate, - if: ->(pages_domain, _) { pages_domain.certificate? }, - using: PagesDomainCertificate do |pages_domain| + if: ->(pages_domain, _) { pages_domain.certificate? }, + using: PagesDomainCertificate do |pages_domain| pages_domain end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e817dcbbc4b..bcf2e6dae1d 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -25,22 +25,7 @@ module API optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end - def present_groups(groups, options = {}) - options = options.reverse_merge( - with: Entities::Group, - current_user: current_user - ) - - groups = groups.with_statistics if options[:statistics] - present paginate(groups), options - end - end - - resource :groups do - desc 'Get a groups list' do - success Entities::Group - end - params do + params :group_list_params do use :statistics_params optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' @@ -50,14 +35,47 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end - get do - find_params = { all_available: params[:all_available], owned: params[:owned] } + + def find_groups(params) + find_params = { + all_available: params[:all_available], + custom_attributes: params[:custom_attributes], + owned: params[:owned] + } + find_params[:parent] = find_group!(params[:id]) if params[:id] + groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present_groups groups, statistics: params[:statistics] && current_user.admin? + groups + end + + def present_groups(params, groups) + options = { + with: Entities::Group, + current_user: current_user, + statistics: params[:statistics] && current_user.admin? + } + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + + resource :groups do + include CustomAttributesEndpoints + + desc 'Get a groups list' do + success Entities::Group + end + params do + use :group_list_params + end + get do + groups = find_groups(params) + present_groups params, groups end desc 'Create a group. Available only for users who can create groups.' do @@ -159,6 +177,17 @@ module API present paginate(projects), with: entity, current_user: current_user end + desc 'Get a list of subgroups in this group.' do + success Entities::Group + end + params do + use :group_list_params + end + get ":id/subgroups" do + groups = find_groups(params) + present_groups params, groups + end + desc 'Transfer a project to the group namespace. Available only for admin.' do success Entities::GroupDetail end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d6df269486a..7f436b69091 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -156,6 +156,11 @@ module API end end + def authenticated_with_full_private_access! + authenticate! + forbidden! unless current_user.full_private_access? + end + def authenticated_as_admin! authenticate! forbidden! unless current_user.admin? @@ -191,6 +196,10 @@ module API not_found! unless user_project.pages_available? end + def require_pages_config_enabled! + not_found! unless Gitlab.config.pages.enabled + end + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end @@ -329,6 +338,7 @@ module API finder_params[:archived] = params[:archived] finder_params[:search] = params[:search] if params[:search] finder_params[:user] = params.delete(:user) if params[:user] + finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] finder_params end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 6bb85dd2619..0d57c822578 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -36,6 +36,18 @@ module API {} end + def fix_git_env_repository_paths(env, repository_path) + if obj_dir_relative = env['GIT_OBJECT_DIRECTORY_RELATIVE'].presence + env['GIT_OBJECT_DIRECTORY'] = File.join(repository_path, obj_dir_relative) + end + + if alt_obj_dirs_relative = env['GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE'].presence + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_obj_dirs_relative.map { |dir| File.join(repository_path, dir) } + end + + env + end + def log_user_activity(actor) commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 6e78ac2c903..451121a4cea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -19,7 +19,9 @@ module API status 200 # Stores some Git-specific env thread-safely - Gitlab::Git::Env.set(parse_env) + env = parse_env + env = fix_git_env_repository_paths(env, repository_path) if project + Gitlab::Git::Env.set(env) actor = if params[:key_id] diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0df41dcc903..74dfd9f96de 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -68,7 +68,7 @@ module API desc: 'Return issues for the given scope: `created-by-me`, `assigned-to-me` or `all`' end get do - issues = find_issues + issues = paginate(find_issues) options = { with: Entities::IssueBasic, @@ -76,7 +76,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -95,7 +95,7 @@ module API get ":id/issues" do group = find_group!(params[:id]) - issues = find_issues(group_id: group.id) + issues = paginate(find_issues(group_id: group.id)) options = { with: Entities::IssueBasic, @@ -103,7 +103,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end end @@ -124,7 +124,7 @@ module API get ":id/issues" do project = find_project!(params[:id]) - issues = find_issues(project_id: project.id) + issues = paginate(find_issues(project_id: project.id)) options = { with: Entities::IssueBasic, @@ -133,7 +133,7 @@ module API issuable_metadata: issuable_meta_data(issues, 'Issue') } - present paginate(issues), options + present issues, options end desc 'Get a single project issue' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 3c1c412ba42..a116ab3c9bd 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -136,7 +136,7 @@ module API authorize_update_builds! build = find_build!(params[:job_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 259f3f34068..d7b613a717e 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -4,7 +4,6 @@ module API before do authenticate! - require_pages_enabled! end after_validation do @@ -29,10 +28,31 @@ module API end end + resource :pages do + before do + require_pages_config_enabled! + authenticated_with_full_private_access! + end + + desc "Get all pages domains" do + success Entities::PagesDomainBasic + end + params do + use :pagination + end + get "domains" do + present paginate(PagesDomain.all), with: Entities::PagesDomainBasic + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: { id: %r{[^/]+} } do + before do + require_pages_enabled! + end + desc 'Get all pages domains' do success Entities::PagesDomain end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index aab7a6c3f93..4cd7e714aa2 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -119,6 +119,8 @@ module API end resource :projects do + include CustomAttributesEndpoints + desc 'Get a list of visible projects for authenticated user' do success Entities::BasicProjectDetails end diff --git a/lib/api/services.rb b/lib/api/services.rb index 6454e475036..bbcc851d07a 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -522,6 +522,12 @@ module API name: :webhook, type: String, desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username to use to post the message' } ], 'teamcity' => [ diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index 69cd12de72c..b201bf77667 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -14,9 +14,11 @@ module API success ::API::Entities::Branch end get ":id/repository/branches" do - branches = user_project.repository.branches.sort_by(&:name) + repository = user_project.repository + branches = repository.branches.sort_by(&:name) + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - present branches, with: ::API::Entities::Branch, project: user_project + present branches, with: ::API::Entities::Branch, project: user_project, merged_branch_names: merged_branch_names end desc 'Delete a branch' diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index f493fd7c7ec..fa0bef39602 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -169,7 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) - authorize!(:update_build, build) + authorize!(:erase_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index ed206a6def0..be360fbfc0c 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -106,7 +106,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = Note.where(commit_id: commit.id).order(:created_at) + notes = commit.notes.order(:created_at) present paginate(notes), with: ::API::Entities::CommitNote end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 3ad09a1b421..b6d273b98c2 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,12 +7,16 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) - # Create namespace dir if missing - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + # Create namespace dir or hashed path if missing + if project.hashed_storage?(:repository) + FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) + else + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace + end if empty_repo?(project) progress.puts "[SKIPPED]".color(:cyan) @@ -42,7 +46,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else @@ -71,7 +75,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " + progress.print " * #{display_repo_path(project)} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -104,7 +108,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " + progress.print " * #{display_repo_path(wiki)} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -185,14 +189,14 @@ module Backup def progress_warn(project, cmd, output) progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{project.full_path} - #{output}".color(:orange) + progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) end def empty_repo?(project_or_wiki) project_or_wiki.repository.expire_exists_cache # protect backups from stale cache project_or_wiki.repository.empty_repo? rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.full_path} - #{e.message}".color(:orange) + progress.puts "Ignoring repository error and continuing backing up project: #{display_repo_path(project_or_wiki)} - #{e.message}".color(:orange) false end @@ -204,5 +208,9 @@ module Backup def progress $progress end + + def display_repo_path(project) + project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path + end end end diff --git a/lib/banzai.rb b/lib/banzai.rb index 35ca234c1ba..5df98f66f3b 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,8 +3,8 @@ module Banzai Renderer.render(text, context) end - def self.render_field(object, field) - Renderer.render_field(object, field) + def self.render_field(object, field, context = {}) + Renderer.render_field(object, field, context) end def self.cache_collection_render(texts_and_contexts) diff --git a/lib/banzai/filter/absolute_link_filter.rb b/lib/banzai/filter/absolute_link_filter.rb new file mode 100644 index 00000000000..1ec6201523f --- /dev/null +++ b/lib/banzai/filter/absolute_link_filter.rb @@ -0,0 +1,34 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that converts relative urls into absolute ones. + class AbsoluteLinkFilter < HTML::Pipeline::Filter + def call + return doc unless context[:only_path] == false + + doc.search('a.gfm').each do |el| + process_link_attr el.attribute('href') + end + + doc + end + + protected + + def process_link_attr(html_attr) + return if html_attr.blank? + return if html_attr.value.start_with?('//') + + uri = URI(html_attr.value) + html_attr.value = absolute_link_attr(uri) if uri.relative? + rescue URI::Error + # noop + end + + def absolute_link_attr(uri) + URI.join(Gitlab.config.gitlab.url, uri).to_s + end + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index a0f7e4e5ad5..9fef386de16 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -311,30 +311,6 @@ module Banzai def project_refs_cache RequestStore[:banzai_project_refs] ||= {} end - - def cached_call(request_store_key, cache_key, path: []) - if RequestStore.active? - cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| - hash[key] = Hash.new { |h, k| h[k] = {} } - end - - cache = cache.dig(*path) if path.any? - - get_or_set_cache(cache, cache_key) { yield } - else - yield - end - end - - def get_or_set_cache(cache, key) - if cache.key?(key) - cache[key] - else - value = yield - cache[key] = value if key.present? - value - end - end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 4fc5f211e84..bb5da310e09 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -56,7 +56,7 @@ module Banzai end def find_milestone_with_finder(project, params) - finder_params = { project_ids: [project.id], order: nil } + finder_params = { project_ids: [project.id], order: nil, state: 'all' } # We don't support IID lookups for group milestones, because IIDs can # clash between group and project milestones. diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index c6ae28adf87..b9d5ecf70ec 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -8,6 +8,8 @@ module Banzai # :project (required) - Current project, ignored if reference is cross-project. # :only_path - Generate path-only links. class ReferenceFilter < HTML::Pipeline::Filter + include RequestStoreReferenceCache + class << self attr_accessor :reference_type end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index afb6e25963c..c7fa8a8119f 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -60,10 +60,14 @@ module Banzai self.class.references_in(text) do |match, username| if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) - elsif namespace = namespaces[username.downcase] - link_to_namespace(namespace, link_content: link_content) || match else - match + cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do + if namespace = namespaces[username.downcase] + link_to_namespace(namespace, link_content: link_content) || match + else + match + end + end end end end @@ -74,7 +78,10 @@ module Banzai # The keys of this Hash are the namespace paths, the values the # corresponding Namespace objects. def namespaces - @namespaces ||= Namespace.where_full_path_in(usernames).index_by(&:full_path).transform_keys(&:downcase) + @namespaces ||= Namespace.eager_load(:owner, :route) + .where_full_path_in(usernames) + .index_by(&:full_path) + .transform_keys(&:downcase) end # Returns all usernames referenced in the current document. diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb deleted file mode 100644 index 2b7c10f1a0e..00000000000 --- a/lib/banzai/note_renderer.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Banzai - module NoteRenderer - # Renders a collection of Note instances. - # - # notes - The notes to render. - # project - The project to use for redacting. - # user - The user viewing the notes. - # path - The request path. - # wiki - The project's wiki. - # git_ref - The current Git reference. - def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil) - renderer = ObjectRenderer.new(project, - user, - requested_path: path, - project_wiki: wiki, - ref: git_ref) - - renderer.render(notes, :note) - end - end -end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index e40556e869c..9bb8ed913d8 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -37,7 +37,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) # rubocop:disable GitlabSecurity/PublicSend + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html(save_options).html_safe) # rubocop:disable GitlabSecurity/PublicSend object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) end end @@ -83,5 +83,10 @@ module Banzai skip_redaction: true ) end + + def save_options + return {} unless base_context[:xhtml] + { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } + end end end diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 131ac3b0eec..dcd52bc03c7 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -3,9 +3,10 @@ module Banzai class PostProcessPipeline < BasePipeline def self.filters FilterArray[ + Filter::RedactorFilter, Filter::RelativeLinkFilter, Filter::IssuableStateFilter, - Filter::RedactorFilter + Filter::AbsoluteLinkFilter ] end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 5f91884a878..5cb9adf52b0 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -32,12 +32,9 @@ module Banzai # Convert a Markdown-containing field on an object into an HTML-safe String # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. - # - # The context to use is managed by the object and cannot be changed. - # Use #render, passing it the field text, if a custom rendering is needed. - def self.render_field(object, field) + def self.render_field(object, field, context = {}) unless object.respond_to?(:cached_markdown_fields) - return cacheless_render_field(object, field) + return cacheless_render_field(object, field, context) end object.refresh_markdown_cache! unless object.cached_html_up_to_date?(field) @@ -46,9 +43,9 @@ module Banzai end # Same as +render_field+, but without consulting or updating the cache field - def self.cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, context = {}) text = object.__send__(field) # rubocop:disable GitlabSecurity/PublicSend - context = object.banzai_render_context(field).merge(options) + context = context.reverse_merge(object.banzai_render_context(field)) if object.respond_to?(:banzai_render_context) cacheless_render(text, context) end diff --git a/lib/banzai/request_store_reference_cache.rb b/lib/banzai/request_store_reference_cache.rb new file mode 100644 index 00000000000..426131442a2 --- /dev/null +++ b/lib/banzai/request_store_reference_cache.rb @@ -0,0 +1,27 @@ +module Banzai + module RequestStoreReferenceCache + def cached_call(request_store_key, cache_key, path: []) + if RequestStore.active? + cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + + cache = cache.dig(*path) if path.any? + + get_or_set_cache(cache, cache_key) { yield } + else + yield + end + end + + def get_or_set_cache(cache, key) + if cache.key?(key) + cache[key] + else + value = yield + cache[key] = value if key.present? + value + end + end + end +end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 6fc1d56d7a0..fd2ac2db0a9 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,7 +2,7 @@ class GroupUrlConstrainer def matches?(request) full_path = request.params[:group_id] || request.params[:id] - return false unless DynamicPathValidator.valid_group_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) Group.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 5bef29eb1da..e90ecb5ec69 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,7 +4,7 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = [namespace_path, project_path].join('/') - return false unless DynamicPathValidator.valid_project_path?(full_path) + return false unless ProjectPathValidator.valid_path?(full_path) # We intentionally allow SELECT(*) here so result of this query can be used # as cache for further Project.find_by_full_path calls within request diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index d16ae7f3f40..b7633aa7cbb 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -2,7 +2,7 @@ class UserUrlConstrainer def matches?(request) full_path = request.params[:username] - return false unless DynamicPathValidator.valid_user_path?(full_path) + return false unless UserPathValidator.valid_path?(full_path) User.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/feature.rb b/lib/feature.rb index 4bd29aed687..ac3bc65c0d5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -5,6 +5,10 @@ class Feature class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature # Using `self.table_name` won't work. ActiveRecord bug? superclass.table_name = 'features' + + def self.feature_names + pluck(:key) + end end class FlipperGate < Flipper::Adapters::ActiveRecord::Gate @@ -22,11 +26,19 @@ class Feature flipper.feature(key) end + def persisted_names + if RequestStore.active? + RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names + else + FlipperFeature.feature_names + end + end + def persisted?(feature) # Flipper creates on-memory features when asked for a not-yet-created one. # If we want to check if a feature has been actually set, we look for it # on the persisted features list. - all.map(&:name).include?(feature.name) + persisted_names.include?(feature.name) end def enabled?(key, thing = nil) diff --git a/lib/github/client.rb b/lib/github/client.rb deleted file mode 100644 index 29bd9c1f39e..00000000000 --- a/lib/github/client.rb +++ /dev/null @@ -1,54 +0,0 @@ -module Github - class Client - TIMEOUT = 60 - DEFAULT_PER_PAGE = 100 - - attr_reader :connection, :rate_limit - - def initialize(options) - @connection = Faraday.new(url: options.fetch(:url, root_endpoint)) do |faraday| - faraday.options.open_timeout = options.fetch(:timeout, TIMEOUT) - faraday.options.timeout = options.fetch(:timeout, TIMEOUT) - faraday.authorization 'token', options.fetch(:token) - faraday.adapter :net_http - faraday.ssl.verify = verify_ssl - end - - @rate_limit = RateLimit.new(connection) - end - - def get(url, query = {}) - exceed, reset_in = rate_limit.get - sleep reset_in if exceed - - Github::Response.new(connection.get(url, { per_page: DEFAULT_PER_PAGE }.merge(query))) - end - - private - - def root_endpoint - custom_endpoint || github_endpoint - end - - def custom_endpoint - github_omniauth_provider.dig('args', 'client_options', 'site') - end - - def verify_ssl - # If there is no config, we're connecting to github.com - # and we should verify ssl. - github_omniauth_provider.fetch('verify_ssl', true) - end - - def github_endpoint - OmniAuth::Strategies::GitHub.default_options[:client_options][:site] - end - - def github_omniauth_provider - @github_omniauth_provider ||= - Gitlab.config.omniauth.providers - .find { |provider| provider.name == 'github' } - .to_h - end - end -end diff --git a/lib/github/collection.rb b/lib/github/collection.rb deleted file mode 100644 index 014b2038c4b..00000000000 --- a/lib/github/collection.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Github - class Collection - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch(url, query = {}) - return [] if url.blank? - - Enumerator.new do |yielder| - loop do - response = client.get(url, query) - response.body.each { |item| yielder << item } - - raise StopIteration unless response.rels.key?(:next) - url = response.rels[:next] - end - end.lazy - end - - private - - def client - @client ||= Github::Client.new(options) - end - end -end diff --git a/lib/github/error.rb b/lib/github/error.rb deleted file mode 100644 index 66d7afaa787..00000000000 --- a/lib/github/error.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Github - RepositoryFetchError = Class.new(StandardError) -end diff --git a/lib/github/import.rb b/lib/github/import.rb deleted file mode 100644 index 8cabbdec940..00000000000 --- a/lib/github/import.rb +++ /dev/null @@ -1,378 +0,0 @@ -require_relative 'error' -require_relative 'import/issue' -require_relative 'import/legacy_diff_note' -require_relative 'import/merge_request' -require_relative 'import/note' - -module Github - class Import - include Gitlab::ShellAdapter - - attr_reader :project, :repository, :repo, :repo_url, :wiki_url, - :options, :errors, :cached, :verbose, :last_fetched_at - - def initialize(project, options = {}) - @project = project - @repository = project.repository - @repo = project.import_source - @repo_url = project.import_url - @wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') - @options = options.reverse_merge(token: project.import_data&.credentials&.fetch(:user)) - @verbose = options.fetch(:verbose, false) - @cached = Hash.new { |hash, key| hash[key] = Hash.new } - @errors = [] - @last_fetched_at = nil - end - - # rubocop: disable Rails/Output - def execute - puts 'Fetching repository...'.color(:aqua) if verbose - setup_and_fetch_repository - puts 'Fetching labels...'.color(:aqua) if verbose - fetch_labels - puts 'Fetching milestones...'.color(:aqua) if verbose - fetch_milestones - puts 'Fetching pull requests...'.color(:aqua) if verbose - fetch_pull_requests - puts 'Fetching issues...'.color(:aqua) if verbose - fetch_issues - puts 'Fetching releases...'.color(:aqua) if verbose - fetch_releases - puts 'Cloning wiki repository...'.color(:aqua) if verbose - fetch_wiki_repository - puts 'Expiring repository cache...'.color(:aqua) if verbose - expire_repository_cache - - errors.empty? - rescue Github::RepositoryFetchError - expire_repository_cache - false - ensure - keep_track_of_errors - end - - private - - def setup_and_fetch_repository - begin - project.ensure_repository - project.repository.add_remote('github', repo_url) - project.repository.set_import_remote_as_mirror('github') - project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') - fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, - Gitlab::Git::RepositoryMirroring::RemoteError, - Gitlab::Shell::Error => e - error(:project, repo_url, e.message) - raise Github::RepositoryFetchError - end - end - - def fetch_remote(forced: false) - @last_fetched_at = Time.now - project.repository.fetch_remote('github', forced: forced) - end - - def fetch_wiki_repository - return if project.wiki.repository_exists? - - wiki_path = project.wiki.disk_path - gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) - rescue Gitlab::Shell::Error => e - # GitHub error message when the wiki repo has not been created, - # this means that repo has wiki enabled, but have no pages. So, - # we can skip the import. - if e.message !~ /repository not exported/ - error(:wiki, wiki_url, e.message) - end - end - - def fetch_labels - url = "/repos/#{repo}/labels" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - begin - representation = Github::Representation::Label.new(raw) - - label = project.labels.find_or_create_by!(title: representation.title) do |label| - label.color = representation.color - end - - cached[:label_ids][representation.title] = label.id - rescue => e - error(:label, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_milestones - url = "/repos/#{repo}/milestones" - - while url - response = Github::Client.new(options).get(url, state: :all) - - response.body.each do |raw| - begin - milestone = Github::Representation::Milestone.new(raw) - next if project.milestones.where(iid: milestone.iid).exists? - - project.milestones.create!( - iid: milestone.iid, - title: milestone.title, - description: milestone.description, - due_date: milestone.due_date, - state: milestone.state, - created_at: milestone.created_at, - updated_at: milestone.updated_at - ) - rescue => e - error(:milestone, milestone.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_pull_requests - url = "/repos/#{repo}/pulls" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) - merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) - next unless merge_request.new_record? && pull_request.valid? - - begin - # If the PR has been created/updated after we last fetched the - # remote, we fetch again to get the up-to-date refs. - fetch_remote if pull_request.updated_at > last_fetched_at - - author_id = user_id(pull_request.author, project.creator_id) - description = format_description(pull_request.description, pull_request.author) - - merge_request.attributes = { - iid: pull_request.iid, - title: pull_request.title, - description: description, - ref_fetched: true, - source_project: pull_request.source_project, - source_branch: pull_request.source_branch_name, - source_branch_sha: pull_request.source_branch_sha, - target_project: pull_request.target_project, - target_branch: pull_request.target_branch_name, - target_branch_sha: pull_request.target_branch_sha, - state: pull_request.state, - milestone_id: milestone_id(pull_request.milestone), - author_id: author_id, - assignee_id: user_id(pull_request.assignee), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - merge_request.save!(validate: false) - merge_request.merge_request_diffs.create - - review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" - fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) - rescue => e - error(:pull_request, pull_request.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def fetch_issues - url = "/repos/#{repo}/issues" - - while url - response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) - - response.body.each { |raw| populate_issue(raw) } - - url = response.rels[:next] - end - end - - def populate_issue(raw) - representation = Github::Representation::Issue.new(raw, options) - - begin - # Every pull request is an issue, but not every issue - # is a pull request. For this reason, "shared" actions - # for both features, like manipulating assignees, labels - # and milestones, are provided within the Issues API. - if representation.pull_request? - return unless representation.labels? || representation.comments? - - merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) - - if representation.labels? - merge_request.update_attribute(:label_ids, label_ids(representation.labels)) - end - - fetch_comments_conditionally(merge_request, representation) - else - return if Issue.exists?(iid: representation.iid, project_id: project.id) - - author_id = user_id(representation.author, project.creator_id) - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = format_description(representation.description, representation.author) - issue.state = representation.state - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = author_id - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save!(validate: false) - - issue.update( - label_ids: label_ids(representation.labels), - assignee_ids: assignee_ids(representation.assignees)) - - fetch_comments_conditionally(issue, representation) - end - rescue => e - error(:issue, representation.url, e.message) - end - end - - def fetch_comments_conditionally(issuable, representation) - if representation.comments? - comments_url = "/repos/#{repo}/issues/#{issuable.iid}/comments" - fetch_comments(issuable, :comment, comments_url) - end - end - - def fetch_comments(noteable, type, url, klass = Note) - while url - comments = Github::Client.new(options).get(url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - representation = Github::Representation::Comment.new(raw, options) - author_id = user_id(representation.author, project.creator_id) - - note = klass.new - note.project_id = project.id - note.noteable = noteable - note.note = format_description(representation.note, representation.author) - note.commit_id = representation.commit_id - note.line_code = representation.line_code - note.author_id = author_id - note.created_at = representation.created_at - note.updated_at = representation.updated_at - note.save!(validate: false) - rescue => e - error(type, representation.url, e.message) - end - end - end - - url = comments.rels[:next] - end - end - - def fetch_releases - url = "/repos/#{repo}/releases" - - while url - response = Github::Client.new(options).get(url) - - response.body.each do |raw| - representation = Github::Representation::Release.new(raw) - next unless representation.valid? - - release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) - next unless release.new_record? - - begin - release.description = representation.description - release.created_at = representation.created_at - release.updated_at = representation.updated_at - release.save!(validate: false) - rescue => e - error(:release, representation.url, e.message) - end - end - - url = response.rels[:next] - end - end - - def label_ids(labels) - labels.map { |label| cached[:label_ids][label.title] }.compact - end - - def assignee_ids(assignees) - assignees.map { |assignee| user_id(assignee) }.compact - end - - def milestone_id(milestone) - return unless milestone.present? - - project.milestones.select(:id).find_by(iid: milestone.iid)&.id - end - - def user_id(user, fallback_id = nil) - return unless user.present? - return cached[:user_ids][user.id] if cached[:user_ids][user.id].present? - - gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) - - cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? - cached[:user_ids][user.id] = gitlab_user_id || fallback_id - end - - def user_id_by_email(email) - return nil unless email - - ::User.find_by_any_email(email)&.id - end - - def user_id_by_external_uid(id) - return nil unless id - - ::User.select(:id) - .joins(:identities) - .merge(::Identity.where(provider: :github, extern_uid: id)) - .first&.id - end - - def format_description(body, author) - return body if cached[:gitlab_user_ids][author.id] - - "*Created by: #{author.username}*\n\n#{body}" - end - - def expire_repository_cache - repository.expire_content_cache if project.repository_exists? - end - - def keep_track_of_errors - return unless errors.any? - - project.update_column(:import_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def error(type, url, message) - errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } - end - end -end diff --git a/lib/github/import/issue.rb b/lib/github/import/issue.rb deleted file mode 100644 index 171f0872666..00000000000 --- a/lib/github/import/issue.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Issue < ::Issue - self.table_name = 'issues' - - self.reset_callbacks :save - self.reset_callbacks :create - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/legacy_diff_note.rb b/lib/github/import/legacy_diff_note.rb deleted file mode 100644 index 18adff560b6..00000000000 --- a/lib/github/import/legacy_diff_note.rb +++ /dev/null @@ -1,12 +0,0 @@ -module Github - class Import - class LegacyDiffNote < ::LegacyDiffNote - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/merge_request.rb b/lib/github/import/merge_request.rb deleted file mode 100644 index c258e5d5e0e..00000000000 --- a/lib/github/import/merge_request.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class MergeRequest < ::MergeRequest - self.table_name = 'merge_requests' - - self.reset_callbacks :create - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/import/note.rb b/lib/github/import/note.rb deleted file mode 100644 index 8cf4f30e6b7..00000000000 --- a/lib/github/import/note.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - class Import - class Note < ::Note - self.table_name = 'notes' - self.store_full_sti_class = false - - self.reset_callbacks :save - self.reset_callbacks :commit - self.reset_callbacks :update - self.reset_callbacks :validate - end - end -end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb deleted file mode 100644 index 884693d093c..00000000000 --- a/lib/github/rate_limit.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - class RateLimit - SAFE_REMAINING_REQUESTS = 100 - SAFE_RESET_TIME = 500 - RATE_LIMIT_URL = '/rate_limit'.freeze - - attr_reader :connection - - def initialize(connection) - @connection = connection - end - - def get - response = connection.get(RATE_LIMIT_URL) - - # GitHub Rate Limit API returns 404 when the rate limit is disabled - return false unless response.status != 404 - - body = Oj.load(response.body, class_cache: false, mode: :compat) - remaining = body.dig('rate', 'remaining').to_i - reset_in = body.dig('rate', 'reset').to_i - exceed = remaining <= SAFE_REMAINING_REQUESTS - - [exceed, reset_in] - end - end -end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb deleted file mode 100644 index c1c9448f305..00000000000 --- a/lib/github/repositories.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Github - class Repositories - attr_reader :options - - def initialize(options) - @options = options - end - - def fetch - Collection.new(options).fetch(repos_url) - end - - private - - def repos_url - '/user/repos' - end - end -end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb deleted file mode 100644 index f26bdbdd546..00000000000 --- a/lib/github/representation/base.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Github - module Representation - class Base - def initialize(raw, options = {}) - @raw = raw - @options = options - end - - def id - raw['id'] - end - - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - - private - - attr_reader :raw, :options - end - end -end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb deleted file mode 100644 index 0087a3d3c4f..00000000000 --- a/lib/github/representation/branch.rb +++ /dev/null @@ -1,55 +0,0 @@ -module Github - module Representation - class Branch < Representation::Base - attr_reader :repository - - def user - raw.dig('user', 'login') || 'unknown' - end - - def repo? - raw['repo'].present? - end - - def repo - return unless repo? - - @repo ||= Github::Representation::Repo.new(raw['repo']) - end - - def ref - raw['ref'] - end - - def sha - raw['sha'] - end - - def short_sha - Commit.truncate_sha(sha) - end - - def valid? - sha.present? && ref.present? - end - - def restore!(name) - repository.create_branch(name, sha) - rescue Gitlab::Git::Repository::InvalidRef => e - Rails.logger.error("#{self.class.name}: Could not restore branch #{name}: #{e}") - end - - def remove!(name) - repository.delete_branch(name) - rescue Gitlab::Git::Repository::DeleteBranchError => e - Rails.logger.error("#{self.class.name}: Could not remove branch #{name}: #{e}") - end - - private - - def repository - @repository ||= options.fetch(:repository) - end - end - end -end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb deleted file mode 100644 index 83bf0b5310d..00000000000 --- a/lib/github/representation/comment.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Github - module Representation - class Comment < Representation::Base - def note - raw['body'] || '' - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def commit_id - raw['commit_id'] - end - - def line_code - return unless on_diff? - - parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) - generate_line_code(parsed_lines.to_a.last) - end - - private - - def generate_line_code(line) - Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) - end - - def on_diff? - diff_hunk.present? - end - - def diff_hunk - raw['diff_hunk'] - end - - def file_path - raw['path'] - end - end - end -end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb deleted file mode 100644 index 768ba3b993c..00000000000 --- a/lib/github/representation/issuable.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Github - module Representation - class Issuable < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user'], options) - end - - def labels? - raw['labels'].any? - end - - def labels - @labels ||= Array(raw['labels']).map do |label| - Github::Representation::Label.new(label, options) - end - end - end - end -end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb deleted file mode 100644 index 4f1a02cb90f..00000000000 --- a/lib/github/representation/issue.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Github - module Representation - class Issue < Representation::Issuable - def state - raw['state'] == 'closed' ? 'closed' : 'opened' - end - - def comments? - raw['comments'] > 0 - end - - def pull_request? - raw['pull_request'].present? - end - - def assigned? - raw['assignees'].present? - end - - def assignees - @assignees ||= Array(raw['assignees']).map do |user| - Github::Representation::User.new(user, options) - end - end - end - end -end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb deleted file mode 100644 index 60aa51f9569..00000000000 --- a/lib/github/representation/label.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Github - module Representation - class Label < Representation::Base - def color - "##{raw['color']}" - end - - def title - raw['name'] - end - end - end -end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb deleted file mode 100644 index 917e6394ad4..00000000000 --- a/lib/github/representation/milestone.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - module Representation - class Milestone < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['description'] - end - - def due_date - raw['due_on'] - end - - def state - raw['state'] == 'closed' ? 'closed' : 'active' - end - end - end -end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb deleted file mode 100644 index 0171179bb0f..00000000000 --- a/lib/github/representation/pull_request.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Github - module Representation - class PullRequest < Representation::Issuable - delegate :sha, to: :source_branch, prefix: true - delegate :sha, to: :target_branch, prefix: true - - def source_project - project - end - - def source_branch_name - # Mimic the "user:branch" displayed in the MR widget, - # i.e. "Request to merge rymai:add-external-mounts into master" - cross_project? ? "#{source_branch.user}:#{source_branch.ref}" : source_branch.ref - end - - def target_project - project - end - - def target_branch_name - target_branch.ref - end - - def state - return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? - return 'closed' if raw['state'] == 'closed' - - 'opened' - end - - def opened? - state == 'opened' - end - - def valid? - source_branch.valid? && target_branch.valid? - end - - def assigned? - raw['assignee'].present? - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee'], options) - end - - private - - def project - @project ||= options.fetch(:project) - end - - def source_branch - @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) - end - - def target_branch - @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) - end - - def cross_project? - return true unless source_branch.repo? - - source_branch.repo.id != target_branch.repo.id - end - end - end -end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb deleted file mode 100644 index e7e4b428c1a..00000000000 --- a/lib/github/representation/release.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Github - module Representation - class Release < Representation::Base - def description - raw['body'] - end - - def tag - raw['tag_name'] - end - - def valid? - !raw['draft'] - end - end - end -end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb deleted file mode 100644 index 6938aa7db05..00000000000 --- a/lib/github/representation/repo.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Github - module Representation - class Repo < Representation::Base - end - end -end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb deleted file mode 100644 index 18591380e25..00000000000 --- a/lib/github/representation/user.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Github - module Representation - class User < Representation::Base - def email - return @email if defined?(@email) - - @email = Github::User.new(username, options).get.fetch('email', nil) - end - - def username - raw['login'] - end - end - end -end diff --git a/lib/github/response.rb b/lib/github/response.rb deleted file mode 100644 index 761c524b553..00000000000 --- a/lib/github/response.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Github - class Response - attr_reader :raw, :headers, :status - - def initialize(response) - @raw = response - @headers = response.headers - @status = response.status - end - - def body - Oj.load(raw.body, class_cache: false, mode: :compat) - end - - def rels - links = headers['Link'].to_s.split(', ').map do |link| - href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures - - [name.to_sym, href] - end - - Hash[*links.flatten] - end - end -end diff --git a/lib/github/user.rb b/lib/github/user.rb deleted file mode 100644 index f88a29e590b..00000000000 --- a/lib/github/user.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Github - class User - attr_reader :username, :options - - def initialize(username, options) - @username = username - @options = options - end - - def get - client.get(user_url).body - end - - private - - def client - @client ||= Github::Client.new(options) - end - - def user_url - "/users/#{username}" - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0ad9285c0ea..cbbc51db99e 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -25,7 +25,7 @@ module Gitlab result = service_request_check(login, password, project) || build_access_token_check(login, password) || - lfs_token_check(login, password) || + lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || user_with_password_for_git(login, password) || @@ -146,7 +146,7 @@ module Gitlab end.flatten.uniq end - def lfs_token_check(login, password) + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) actor = @@ -163,6 +163,8 @@ module Gitlab authentication_abilities = if token_handler.user? full_authentication_abilities + elsif token_handler.deploy_key_pushable?(project) + read_write_authentication_abilities else read_authentication_abilities end @@ -208,10 +210,15 @@ module Gitlab ] end - def full_authentication_abilities + def read_write_authentication_abilities read_authentication_abilities + [ :push_code, - :create_container_image, + :create_container_image + ] + end + + def full_authentication_abilities + read_write_authentication_abilities + [ :admin_container_image ] end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index c88eb9783ed..67a39d28944 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -51,10 +51,20 @@ module Gitlab FROM projects WHERE forked_project_links.forked_from_project_id = projects.id ) + AND NOT EXISTS ( + SELECT true + FROM forked_project_links AS parent_links + WHERE parent_links.forked_to_project_id = forked_project_links.forked_from_project_id + AND NOT EXISTS ( + SELECT true + FROM projects + WHERE parent_links.forked_from_project_id = projects.id + ) + ) AND forked_project_links.id BETWEEN #{start_id} AND #{end_id} MISSING_MEMBERS - ForkNetworkMember.count_by_sql(count_sql) > 0 + ForkedProjectLink.count_by_sql(count_sql) > 0 end def log(message) diff --git a/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb new file mode 100644 index 00000000000..7e109e96e73 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb @@ -0,0 +1,30 @@ +module Gitlab + module BackgroundMigration + class PopulateMergeRequestsLatestMergeRequestDiffId + BATCH_SIZE = 1_000 + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include ::EachBatch + end + + def perform(start_id, stop_id) + update = ' + latest_merge_request_diff_id = ( + SELECT MAX(id) + FROM merge_request_diffs + WHERE merge_requests.id = merge_request_diffs.merge_request_id + )'.squish + + MergeRequest + .where(id: start_id..stop_id) + .where(latest_merge_request_diff_id: nil) + .each_batch(of: BATCH_SIZE) do |relation| + + relation.update_all(update) + end + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb new file mode 100644 index 00000000000..196de667805 --- /dev/null +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -0,0 +1,101 @@ +module Gitlab + module BareRepositoryImport + class Importer + NoAdminError = Class.new(StandardError) + + def self.execute(import_path) + import_path << '/' unless import_path.ends_with?('/') + repos_to_import = Dir.glob(import_path + '**/*.git') + + unless user = User.admins.order_id_asc.first + raise NoAdminError.new('No admin user found to import repositories') + end + + repos_to_import.each do |repo_path| + bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) + + if bare_repo.hashed? || bare_repo.wiki? + log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) + + next + end + + log "Processing #{repo_path}".color(:yellow) + + new(user, bare_repo).create_project_if_needed + end + end + + attr_reader :user, :project_name, :bare_repo + + delegate :log, to: :class + delegate :project_name, :project_full_path, :group_path, :repo_path, :wiki_path, to: :bare_repo + + def initialize(user, bare_repo) + @user = user + @bare_repo = bare_repo + end + + def create_project_if_needed + if project = Project.find_by_full_path(project_full_path) + log " * #{project.name} (#{project_full_path}) exists" + + return project + end + + create_project + end + + private + + def create_project + group = find_or_create_groups + + project = Projects::CreateService.new(user, + name: project_name, + path: project_name, + skip_disk_validation: true, + namespace_id: group&.id).execute + + if project.persisted? && mv_repo(project) + log " * Created #{project.name} (#{project_full_path})".color(:green) + + ProjectCacheWorker.perform_async(project.id) + else + log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) + log " Errors: #{project.errors.messages}".color(:red) if project.errors.any? + end + + project + end + + def mv_repo(project) + FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git')) + + if bare_repo.wiki_exists? + FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git')) + end + + true + rescue => e + log " * Failed to move repo: #{e.message}".color(:red) + + false + end + + def find_or_create_groups + return nil unless group_path.present? + + log " * Using namespace: #{group_path}" + + Groups::NestedCreateService.new(user, group_path: group_path).execute + end + + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb new file mode 100644 index 00000000000..8574ac6eb30 --- /dev/null +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -0,0 +1,42 @@ +module Gitlab + module BareRepositoryImport + class Repository + attr_reader :group_path, :project_name, :repo_path + + def initialize(root_path, repo_path) + @root_path = root_path + @repo_path = repo_path + + # Split path into 'all/the/namespaces' and 'project_name' + @group_path, _, @project_name = repo_relative_path.rpartition('/') + end + + def wiki_exists? + File.exist?(wiki_path) + end + + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def wiki_path + @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') + end + + def hashed? + @hashed ||= group_path.start_with?('@hashed') + end + + def project_full_path + @project_full_path ||= "#{group_path}/#{project_name}" + end + + private + + def repo_relative_path + # Remove root path and `.git` at the end + repo_path[@root_path.size...-4] + end + end + end +end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb deleted file mode 100644 index 1d98d187805..00000000000 --- a/lib/gitlab/bare_repository_importer.rb +++ /dev/null @@ -1,97 +0,0 @@ -module Gitlab - class BareRepositoryImporter - NoAdminError = Class.new(StandardError) - - def self.execute - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - git_base_path = repository_storage['path'] - repos_to_import = Dir.glob(git_base_path + '/**/*.git') - - repos_to_import.each do |repo_path| - if repo_path.end_with?('.wiki.git') - log " * Skipping wiki repo" - next - end - - log "Processing #{repo_path}".color(:yellow) - - repo_relative_path = repo_path[repository_storage['path'].length..-1] - .sub(/^\//, '') # Remove leading `/` - .sub(/\.git$/, '') # Remove `.git` at the end - new(storage_name, repo_relative_path).create_project_if_needed - end - end - end - - attr_reader :storage_name, :full_path, :group_path, :project_path, :user - delegate :log, to: :class - - def initialize(storage_name, repo_path) - @storage_name = storage_name - @full_path = repo_path - - unless @user = User.admins.order_id_asc.first - raise NoAdminError.new('No admin user found to import repositories') - end - - @group_path, @project_path = File.split(repo_path) - @group_path = nil if @group_path == '.' - end - - def create_project_if_needed - if project = Project.find_by_full_path(full_path) - log " * #{project.name} (#{full_path}) exists" - return project - end - - create_project - end - - private - - def create_project - group = find_or_create_group - - project_params = { - name: project_path, - path: project_path, - repository_storage: storage_name, - namespace_id: group&.id, - skip_disk_validation: true - } - - project = Projects::CreateService.new(user, project_params).execute - - if project.persisted? - log " * Created #{project.name} (#{full_path})".color(:green) - ProjectCacheWorker.perform_async(project.id) - else - log " * Failed trying to create #{project.name} (#{full_path})".color(:red) - log " Errors: #{project.errors.messages}".color(:red) - end - - project - end - - def find_or_create_group - return nil unless group_path - - if namespace = Namespace.find_by_full_path(group_path) - log " * Namespace #{group_path} exists.".color(:green) - return namespace - end - - log " * Creating Group: #{group_path}" - Groups::NestedCreateService.new(user, group_path: group_path).execute - end - - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - # - # rubocop:disable Rails/Output - def self.log(message) - puts message - end - # rubocop:enable Rails/Output - end -end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index b6805230348..ef92fc5a0a0 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -12,7 +12,8 @@ module Gitlab change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', delete_protected_tag: 'Protected tags cannot be deleted.', - create_protected_tag: 'You are not allowed to create this tag as it is protected.' + create_protected_tag: 'You are not allowed to create this tag as it is protected.', + lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze attr_reader :user_access, :project, :skip_authorization, :protocol @@ -36,6 +37,7 @@ module Gitlab push_checks branch_checks tag_checks + lfs_objects_exist_check true end @@ -136,6 +138,14 @@ module Gitlab def matching_merge_request? Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match? end + + def lfs_objects_exist_check + lfs_check = Checks::LfsIntegrity.new(project, @newrev) + + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] + end + end end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb new file mode 100644 index 00000000000..f7276a380dc --- /dev/null +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -0,0 +1,27 @@ +module Gitlab + module Checks + class LfsIntegrity + REV_LIST_OBJECT_LIMIT = 2_000 + + def initialize(project, newrev) + @project = project + @newrev = newrev + end + + def objects_missing? + return false unless @newrev && @project.lfs_enabled? + + new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev).new_pointers(object_limit: REV_LIST_OBJECT_LIMIT) + + return false unless new_lfs_pointers.present? + + existing_count = @project.lfs_storage_project + .lfs_objects + .where(oid: new_lfs_pointers.map(&:lfs_oid)) + .count + + existing_count != new_lfs_pointers.count + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index d71e63e73eb..dc90f398c7e 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -8,7 +8,7 @@ module Gitlab end def icon - 'warning' + 'status_warning' end def group diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index dfd17e35707..f07fd1dfdda 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -43,7 +43,7 @@ module Gitlab if thread thread.wakeup if thread.alive? - thread.join + thread.join unless Thread.current == thread @thread = nil end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 43a00d6cedb..cd7b4c043da 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -108,20 +108,41 @@ module Gitlab end end - def self.bulk_insert(table, rows) + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows, this only works on PostgreSQL. + def self.bulk_insert(table, rows, return_ids: false) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } + return_ids = false if mysql? tuples = rows.map do |row| row.values_at(*keys).map { |value| connection.quote(value) } end - connection.execute <<-EOF + sql = <<-EOF INSERT INTO #{table} (#{columns.join(', ')}) VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF + + if return_ids + sql << 'RETURNING id' + end + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end end def self.sanitize_timestamp(timestamp) diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index aee3981e79a..9f76967fc77 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,28 +6,36 @@ module Gitlab if Database.postgresql? 'information_schema.role_table_grants' else - 'mysql.user' + 'information_schema.schema_privileges' end - def self.scope_to_current_user - if Database.postgresql? - where('grantee = user') - else - where("CONCAT(User, '@', Host) = current_user()") - end - end - # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) priv = if Database.postgresql? where(privilege_type: 'TRIGGER', table_name: table) + .where('grantee = user') else - where(Trigger_priv: 'Y') + queries = [ + Grant.select(1) + .from('information_schema.user_privileges') + .where("PRIVILEGE_TYPE = 'SUPER'") + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), + + Grant.select(1) + .from('information_schema.schema_privileges') + .where("PRIVILEGE_TYPE = 'TRIGGER'") + .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) + .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") + ] + + union = SQL::Union.new(queries).to_sql + + Grant.from("(#{union}) privs") end - priv.scope_to_current_user.any? + priv.any? end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 0ea534a5fd0..efc2e46d289 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -193,7 +193,7 @@ module Gitlab # Repository is initially cloned with a depth of 20 so we need to fetch # deeper in the case the branch has more than 20 commits on top of master fetch(branch: branch, depth: depth) - fetch(branch: 'master', depth: depth) + fetch(branch: 'master', depth: depth, remote: DEFAULT_CE_PROJECT_URL) merge_base_found? end @@ -201,10 +201,10 @@ module Gitlab raise "\n#{branch} is too far behind master, please rebase it!\n" unless success end - def fetch(branch:, depth:) + def fetch(branch:, depth:, remote: 'origin') step( "Fetching deeper...", - %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + %W[git fetch --depth=#{depth} --prune #{remote} +refs/heads/#{branch}:refs/remotes/origin/#{branch}] ) do |output, status| raise "Fetch failed: #{output}" unless status.zero? end diff --git a/lib/gitlab/gcp/model.rb b/lib/gitlab/gcp/model.rb deleted file mode 100644 index 195391f0e3c..00000000000 --- a/lib/gitlab/gcp/model.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Gitlab - module Gcp - module Model - def table_name_prefix - "gcp_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) - end - end - end -end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ab94ba8a73a..e36d5410431 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -72,7 +72,7 @@ module Gitlab # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, # it would be created from `start_branch_name`. - # If `start_project` is passed, and the branch doesn't exist, + # If `start_repository` is passed, and the branch doesn't exist, # it would try to find the commits from it instead of current repository. def with_branch( branch_name, @@ -80,15 +80,13 @@ module Gitlab start_repository: repository, &block) - # Refactoring aid - unless start_repository.is_a?(Gitlab::Git::Repository) - raise "expected a Gitlab::Git::Repository, got #{start_repository}" - end + Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) start_branch_name = nil if start_repository.empty_repo? if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.full_path}" + raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" end update_branch_with_hooks(branch_name) do diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb new file mode 100644 index 00000000000..3685aa20669 --- /dev/null +++ b/lib/gitlab/git/remote_repository.rb @@ -0,0 +1,82 @@ +module Gitlab + module Git + # + # When a Gitaly call involves two repositories instead of one we cannot + # assume that both repositories are on the same Gitaly server. In this + # case we need to make a distinction between the repository that the + # call is being made on (a Repository instance), and the "other" + # repository (a RemoteRepository instance). This is the reason why we + # have the RemoteRepository class in Gitlab::Git. + # + # When you make changes, be aware that gitaly-ruby sub-classes this + # class. + # + class RemoteRepository + attr_reader :path, :relative_path, :gitaly_repository + + def initialize(repository) + @relative_path = repository.relative_path + @gitaly_repository = repository.gitaly_repository + + # These instance variables will not be available in gitaly-ruby, where + # we have no disk access to this repository. + @repository = repository + @path = repository.path + end + + def empty_repo? + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.empty_repo? + end + + def commit_id(revision) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.commit(revision)&.sha + end + + def branch_exists?(name) + # We will override this implementation in gitaly-ruby because we cannot + # use '@repository' there. + @repository.branch_exists?(name) + end + + # Compares self to a Gitlab::Git::Repository. This implementation uses + # 'self.gitaly_repository' so that it will also work in the + # GitalyRemoteRepository subclass defined in gitaly-ruby. + def same_repository?(other_repository) + gitaly_repository.storage_name == other_repository.storage && + gitaly_repository.relative_path == other_repository.relative_path + end + + def fetch_env + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_client.address(storage) + gitaly_token = gitaly_client.token(storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + env + end + + private + + # Must return an object that responds to 'address' and 'storage'. + def gitaly_client + Gitlab::GitalyClient + end + + def storage + gitaly_repository.storage_name + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 182ffc96ef9..cfb88a0c12b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -58,7 +58,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -66,7 +66,6 @@ module Gitlab @storage = storage @relative_path = relative_path @gl_repository = gl_repository - @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -105,7 +104,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else @@ -920,6 +919,11 @@ module Gitlab false end + # Returns true if a remote exists. + def remote_exists?(name) + rugged.remotes[name].present? + end + # Update the specified remote using the values in the +options+ hash # # Example @@ -1009,23 +1013,22 @@ module Gitlab def with_repo_branch_commit(start_repository, start_branch_name) Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) return yield nil if start_repository.empty_repo? - if start_repository == self + if start_repository.same_repository?(self) yield commit(start_branch_name) else - start_commit = start_repository.commit(start_branch_name) - - return yield nil unless start_commit + start_commit_id = start_repository.commit_id(start_branch_name) - sha = start_commit.sha + return yield nil unless start_commit_id - if branch_commit = commit(sha) + if branch_commit = commit(start_commit_id) yield branch_commit else with_repo_tmp_commit( - start_repository, start_branch_name, sha) do |tmp_commit| + start_repository, start_branch_name, start_commit_id) do |tmp_commit| yield tmp_commit end end @@ -1044,7 +1047,7 @@ module Gitlab delete_refs(tmp_ref) if tmp_ref end - def fetch_source_branch(source_repository, source_branch, local_ref) + def fetch_source_branch!(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) @@ -1082,6 +1085,9 @@ module Gitlab end def fetch_ref(source_repository, source_ref:, target_ref:) + Gitlab::Git.check_namespace!(source_repository) + source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| if is_enabled gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) @@ -1615,22 +1621,9 @@ module Gitlab end def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) - gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) - gitaly_address = gitaly_resolver.address(source_repository.storage) - gitaly_token = gitaly_resolver.token(source_repository.storage) - - request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) - env = { - 'GITALY_ADDRESS' => gitaly_address, - 'GITALY_PAYLOAD' => request.to_json, - 'GITALY_WD' => Dir.pwd, - 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" - } - env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? - args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) - run_git(args, env: env) + run_git(args, env: source_repository.fetch_env) end def gitaly_ff_merge(user, source_sha, target_branch) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index fe901d049d4..022d1f249a9 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -48,15 +48,24 @@ module Gitlab end def update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) - nil + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + if is_enabled + gitaly_update_page(page_path, title, format, content, commit_details) + gollum_wiki.clear_cache + else + gollum_update_page(page_path, title, format, content, commit_details) + end + end end def pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| + if is_enabled + gitaly_get_all_pages + else + gollum_get_all_pages + end + end end def page(title:, version: nil, dir: nil) @@ -149,6 +158,14 @@ module Gitlab nil end + def gollum_update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + def gollum_find_page(title:, version: nil, dir: nil) if version version = Gitlab::Git::Commit.find(@repository, version).id @@ -168,10 +185,18 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end + def gollum_get_all_pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end + def gitaly_update_page(page_path, title, format, content, commit_details) + gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) + end + def gitaly_delete_page(page_path, commit_details) gitaly_wiki_client.delete_page(page_path, commit_details) end @@ -189,6 +214,12 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end + + def gitaly_get_all_pages + gitaly_wiki_client.get_all_pages.map do |wiki_page, version| + Gitlab::Git::WikiPage.new(wiki_page, version) + end + end end end end diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb new file mode 100644 index 00000000000..198a1de91c7 --- /dev/null +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + # This module expects an `ATTRS` const to be defined on the subclass + # See GitalyClient::WikiFile for an example + module AttributesBag + extend ActiveSupport::Concern + + included do + attr_accessor(*const_get(:ATTRS)) + end + + def initialize(params) + params = params.with_indifferent_access + + attributes.each do |attr| + instance_variable_set("@#{attr}", params[attr]) + end + end + + def ==(other) + attributes.all? do |field| + instance_variable_get("@#{field}") == other.instance_variable_get("@#{field}") + end + end + + def attributes + self.class.const_get(:ATTRS) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 54df6304865..d98a0ce988f 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,21 +1,9 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze + ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params.each do |key, val| - public_send(:"#{key}=", val) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def ==(other) - FIELDS.all? do |field| - public_send(field) == other.public_send(field) # rubocop:disable GitlabSecurity/PublicSend - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index 65d81dc5d46..da243ee2d1a 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -12,7 +12,7 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? - diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::ATTRS) # gRPC uses frozen strings by default, and we need to have an unfrozen string as it # gets processed further down the line. So we unfreeze the first chunk of the patch # in case it's the only chunk we receive for this diff. diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb index a2e415864e6..47c60c92484 100644 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ b/lib/gitlab/gitaly_client/wiki_file.rb @@ -1,17 +1,9 @@ module Gitlab module GitalyClient class WikiFile - FIELDS = %i(name mime_type path raw_data).freeze + ATTRS = %i(name mime_type path raw_data).freeze - attr_accessor(*FIELDS) - - def initialize(params) - params = params.with_indifferent_access - - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end - end + include AttributesBag end end end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 8226278d5f6..7339468e911 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -1,16 +1,16 @@ module Gitlab module GitalyClient class WikiPage - FIELDS = %i(title format url_path path name historical raw_data).freeze + ATTRS = %i(title format url_path path name historical raw_data).freeze - attr_accessor(*FIELDS) + include AttributesBag def initialize(params) - params = params.with_indifferent_access + super - FIELDS.each do |field| - instance_variable_set("@#{field}", params[field]) - end + # All gRPC strings in a response are frozen, so we get an unfrozen + # version here so appending to `raw_data` doesn't blow up. + @raw_data = @raw_data.dup end def historical? diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 15f0f30d303..8f05f40365e 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -37,6 +37,31 @@ module Gitlab end end + def update_page(page_path, title, format, content, commit_details) + request = Gitaly::WikiUpdatePageRequest.new( + repository: @gitaly_repo, + page_path: GitalyClient.encode(page_path), + title: GitalyClient.encode(title), + format: format.to_s, + commit_details: gitaly_commit_details(commit_details) + ) + + strio = StringIO.new(content) + + enum = Enumerator.new do |y| + until strio.eof? + chunk = strio.read(MAX_MSG_SIZE) + request.content = GitalyClient.encode(chunk) + + y.yield request + + request = Gitaly::WikiUpdatePageRequest.new + end + end + + GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum) + end + def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, @@ -56,28 +81,23 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) - wiki_page = version = nil - response.each do |message| - page = message.page - next unless page + wiki_page_from_iterator(response) + end - if wiki_page - wiki_page.raw_data << page.raw_data - else - wiki_page = GitalyClient::WikiPage.new(page.to_h) - # All gRPC strings in a response are frozen, so we get - # an unfrozen version here so appending in the else clause below doesn't blow up. - wiki_page.raw_data = wiki_page.raw_data.dup + def get_all_pages + request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request) + pages = [] - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) - end + loop do + page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } + + break unless page && version + pages << [page, version] end - [wiki_page, version] + pages end def find_file(name, revision) @@ -108,6 +128,35 @@ module Gitlab private + # If a block is given and the yielded value is true, iteration will be + # stopped early at that point; else the iterator is consumed entirely. + # The iterator is traversed with `next` to allow resuming the iteration. + def wiki_page_from_iterator(iterator) + wiki_page = version = nil + + while message = iterator.next + break if block_given? && yield(message) + + page = message.page + next unless page + + if wiki_page + wiki_page.raw_data << page.raw_data + else + wiki_page = GitalyClient::WikiPage.new(page.to_h) + + version = Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, page.version.commit), + page.version.format + ) + end + end + + [wiki_page, version] + rescue StopIteration + [wiki_page, version] + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: GitalyClient.encode(commit_details.name), diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb new file mode 100644 index 00000000000..d2ae4c1255e --- /dev/null +++ b/lib/gitlab/github_import.rb @@ -0,0 +1,34 @@ +module Gitlab + module GithubImport + def self.new_client_for(project, token: nil, parallel: true) + token_to_use = token || project.import_data&.credentials&.fetch(:user) + + Client.new(token_to_use, parallel: parallel) + end + + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + def self.insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + + # Returns the ID of the ghost user. + def self.ghost_user_id + key = 'github-import/ghost-user-id' + + Caching.read_integer(key) || Caching.write(key, User.select(:id).ghost.id) + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb new file mode 100644 index 00000000000..147597289cf --- /dev/null +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module BulkImporting + # Builds and returns an Array of objects to bulk insert into the + # database. + # + # enum - An Enumerable that returns the objects to turn into database + # rows. + def build_database_rows(enum) + enum.each_with_object([]) do |(object, _), rows| + rows << build(object) unless already_imported?(object) + end + end + + # Bulk inserts the given rows into the database. + def bulk_insert(model, rows, batch_size: 100) + rows.each_slice(batch_size) do |slice| + Gitlab::Database.bulk_insert(model.table_name, slice) + end + end + end + end +end diff --git a/lib/gitlab/github_import/caching.rb b/lib/gitlab/github_import/caching.rb new file mode 100644 index 00000000000..b08f133794f --- /dev/null +++ b/lib/gitlab/github_import/caching.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Caching + # The default timeout of the cache keys. + TIMEOUT = 24.hours.to_i + + WRITE_IF_GREATER_SCRIPT = <<-EOF.strip_heredoc.freeze + local key, value, ttl = KEYS[1], tonumber(ARGV[1]), ARGV[2] + local existing = tonumber(redis.call("get", key)) + + if existing == nil or value > existing then + redis.call("set", key, value) + redis.call("expire", key, ttl) + return true + else + return false + end + EOF + + # Reads a cache key. + # + # If the key exists and has a non-empty value its TTL is refreshed + # automatically. + # + # raw_key - The cache key to read. + # timeout - The new timeout of the key if the key is to be refreshed. + def self.read(raw_key, timeout: TIMEOUT) + key = cache_key_for(raw_key) + value = Redis::Cache.with { |redis| redis.get(key) } + + if value.present? + # We refresh the expiration time so frequently used keys stick + # around, removing the need for querying the database as much as + # possible. + # + # A key may be empty when we looked up a GitHub user (for example) but + # did not find a matching GitLab user. In that case we _don't_ want to + # refresh the TTL so we automatically pick up the right data when said + # user were to register themselves on the GitLab instance. + Redis::Cache.with { |redis| redis.expire(key, timeout) } + end + + value + end + + # Reads an integer from the cache, or returns nil if no value was found. + # + # See Caching.read for more information. + def self.read_integer(raw_key, timeout: TIMEOUT) + value = read(raw_key, timeout: timeout) + + value.to_i if value.present? + end + + # Sets a cache key to the given value. + # + # key - The cache key to write. + # value - The value to set. + # timeout - The time after which the cache key should expire. + def self.write(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.set(key, value, ex: timeout) + end + + value + end + + # Adds a value to a set. + # + # raw_key - The key of the set to add the value to. + # value - The value to add to the set. + # timeout - The new timeout of the key. + def self.set_add(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.sadd(key, value) + m.expire(key, timeout) + end + end + end + + # Returns true if the given value is present in the set. + # + # raw_key - The key of the set to check. + # value - The value to check for. + def self.set_includes?(raw_key, value) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.sismember(key, value) + end + end + + # Sets multiple keys to a given value. + # + # mapping - A Hash mapping the cache keys to their values. + # timeout - The time after which the cache key should expire. + def self.write_multiple(mapping, timeout: TIMEOUT) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + multi.set(cache_key_for(raw_key), value, ex: timeout) + end + end + end + end + + # Sets the expiration time of a key. + # + # raw_key - The key for which to change the timeout. + # timeout - The new timeout. + def self.expire(raw_key, timeout) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.expire(key, timeout) + end + end + + # Sets a key to the given integer but only if the existing value is + # smaller than the given value. + # + # This method uses a Lua script to ensure the read and write are atomic. + # + # raw_key - The key to set. + # value - The new value for the key. + # timeout - The key timeout in seconds. + # + # Returns true when the key was overwritten, false otherwise. + def self.write_if_greater(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + val = Redis::Cache.with do |redis| + redis + .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) + end + + val ? true : false + end + + def self.cache_key_for(raw_key) + "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0550f9695bd..5da9befa08e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -1,147 +1,216 @@ +# frozen_string_literal: true + module Gitlab module GithubImport + # HTTP client for interacting with the GitHub API. + # + # This class is basically a fancy wrapped around Octokit while adding some + # functionality to deal with rate limiting and parallel imports. Usage is + # mostly the same as Octokit, for example: + # + # client = GithubImport::Client.new('hunter2') + # + # client.labels.each do |label| + # puts label.name + # end class Client - GITHUB_SAFE_REMAINING_REQUESTS = 100 - GITHUB_SAFE_SLEEP_TIME = 500 + attr_reader :octokit + + # A single page of data and the corresponding page number. + Page = Struct.new(:objects, :number) + + # The minimum number of requests we want to keep available. + # + # We don't use a value of 0 as multiple threads may be using the same + # token in parallel. This could result in all of them hitting the GitHub + # rate limit at once. The threshold is put in place to not hit the limit + # in most cases. + RATE_LIMIT_THRESHOLD = 50 + + # token - The GitHub API token to use. + # + # per_page - The number of objects that should be displayed per page. + # + # parallel - When set to true hitting the rate limit will result in a + # dedicated error being raised. When set to `false` we will + # instead just `sleep()` until the rate limit is reset. Setting + # this value to `true` for parallel importing is crucial as + # otherwise hitting the rate limit will result in a thread + # being blocked in a `sleep()` call for up to an hour. + def initialize(token, per_page: 100, parallel: true) + @octokit = Octokit::Client.new( + access_token: token, + per_page: per_page, + api_endpoint: api_endpoint + ) - attr_reader :access_token, :host, :api_version + @octokit.connection_options[:ssl] = { verify: verify_ssl } - def initialize(access_token, host: nil, api_version: 'v3') - @access_token = access_token - @host = host.to_s.sub(%r{/+\z}, '') - @api_version = api_version - @users = {} + @parallel = parallel + end - if access_token - ::Octokit.auto_paginate = false - end + def parallel? + @parallel end - def api - @api ||= ::Octokit::Client.new( - access_token: access_token, - api_endpoint: api_endpoint, - # If there is no config, we're connecting to github.com and we - # should verify ssl. - connection_options: { - ssl: { verify: config ? config['verify_ssl'] : true } - } - ) + # Returns the details of a GitHub user. + # + # username - The username of the user. + def user(username) + with_rate_limit { octokit.user(username) } end - def client - unless config - raise Projects::ImportService::Error, - 'OAuth configuration for GitHub missing.' - end + # Returns the details of a GitHub repository. + # + # name - The path (in the form `owner/repository`) of the repository. + def repository(name) + with_rate_limit { octokit.repo(name) } + end - @client ||= ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + def labels(*args) + each_object(:labels, *args) end - def authorize_url(redirect_uri) - client.auth_code.authorize_url({ - redirect_uri: redirect_uri, - scope: "repo, user, user:email" - }) + def milestones(*args) + each_object(:milestones, *args) end - def get_token(code) - client.auth_code.get_token(code).token + def releases(*args) + each_object(:releases, *args) end - def method_missing(method, *args, &block) - if api.respond_to?(method) - request(method, *args, &block) - else - super(method, *args, &block) + # Fetches data from the GitHub API and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the Octokit method. + # + # rubocop: disable GitlabSecurity/PublicSend + def each_page(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? + + page = + if args.last.is_a?(Hash) && args.last[:page] + args.last[:page] + else + 1 + end + + collection = with_rate_limit { octokit.public_send(method, *args) } + next_url = octokit.last_response.rels[:next] + + yield Page.new(collection, page) + + while next_url + response = with_rate_limit { next_url.get } + next_url = response.rels[:next] + + yield Page.new(response.data, page += 1) end end - def respond_to?(method) - api.respond_to?(method) || super + # Iterates over all of the objects for the given method (e.g. `:labels`). + # + # method - The method to send to Octokit for querying data. + # args - Any arguments to pass to the Octokit method. + def each_object(method, *args, &block) + return to_enum(__method__, method, *args) unless block_given? + + each_page(method, *args) do |page| + page.objects.each do |object| + yield object + end + end end - def user(login) - return nil unless login.present? - return @users[login] if @users.key?(login) + # Yields the supplied block, responding to any rate limit errors. + # + # The exact strategy used for handling rate limiting errors depends on + # whether we are running in parallel mode or not. For more information see + # `#rate_or_wait_for_rate_limit`. + def with_rate_limit + return yield unless rate_limiting_enabled? - @users[login] = api.user(login) - end + request_count_counter.increment - private + raise_or_wait_for_rate_limit unless requests_remaining? - def api_endpoint - if host.present? && api_version.present? - "#{host}/api/#{api_version}" - else - github_options[:site] + begin + yield + rescue Octokit::TooManyRequests + raise_or_wait_for_rate_limit + + # This retry will only happen when running in sequential mode as we'll + # raise an error in parallel mode. + retry end end - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + # Returns `true` if we're still allowed to perform API calls. + def requests_remaining? + remaining_requests > RATE_LIMIT_THRESHOLD + end + + def remaining_requests + octokit.rate_limit.remaining end - def github_options - if config - config["args"]["client_options"].deep_symbolize_keys + def raise_or_wait_for_rate_limit + rate_limit_counter.increment + + if parallel? + raise RateLimitError else - OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + sleep(rate_limit_resets_in) end end - def rate_limit - api.rate_limit! - # GitHub Rate Limit API returns 404 when the rate limit is - # disabled. In this case we just want to return gracefully - # instead of spitting out an error. - rescue Octokit::NotFound - nil + def rate_limit_resets_in + # We add a few seconds to the rate limit so we don't _immediately_ + # resume when the rate limit resets as this may result in us performing + # a request before GitHub has a chance to reset the limit. + octokit.rate_limit.resets_in + 5 end - def has_rate_limit? - return @has_rate_limit if defined?(@has_rate_limit) - - @has_rate_limit = rate_limit.present? + def rate_limiting_enabled? + @rate_limiting_enabled ||= api_endpoint.include?('.github.com') end - def rate_limit_exceed? - has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + def api_endpoint + custom_api_endpoint || default_api_endpoint end - def rate_limit_sleep_time - rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + def custom_api_endpoint + github_omniauth_provider.dig('args', 'client_options', 'site') end - def request(method, *args, &block) - sleep rate_limit_sleep_time if rate_limit_exceed? + def default_api_endpoint + OmniAuth::Strategies::GitHub.default_options[:client_options][:site] + end - data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend - return data unless data.is_a?(Array) + def verify_ssl + github_omniauth_provider.fetch('verify_ssl', true) + end - last_response = api.last_response + def github_omniauth_provider + @github_omniauth_provider ||= + Gitlab.config.omniauth.providers + .find { |provider| provider.name == 'github' } + .to_h + end - if block_given? - yield data - # api.last_response could change while we're yielding (e.g. fetching labels for each PR) - # so we cache our own last response - each_response_page(last_response, &block) - else - each_response_page(last_response) { |page| data.concat(page) } - data - end + def rate_limit_counter + @rate_limit_counter ||= Gitlab::Metrics.counter( + :github_importer_rate_limit_hits, + 'The number of times we hit the GitHub rate limit when importing projects' + ) end - def each_response_page(last_response) - while last_response.rels[:next] - sleep rate_limit_sleep_time if rate_limit_exceed? - last_response = last_response.rels[:next].get - yield last_response.data if last_response.data.is_a?(Array) - end + def request_count_counter + @request_counter ||= Gitlab::Metrics.counter( + :github_importer_request_count, + 'The number of GitHub API calls performed when importing projects' + ) end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb new file mode 100644 index 00000000000..8274f37d358 --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (mr_id = find_merge_request_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: 'MergeRequest', + noteable_id: mr_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + commit_id: note.commit_id, + line_code: note.line_code, + type: 'LegacyDiffNote', + created_at: note.created_at, + updated_at: note.updated_at, + st_diff: note.diff_hash.to_yaml + } + + # It's possible that during an import we'll insert tens of thousands + # of diff notes. If we were to use the Note/LegacyDiffNote model here + # we'd also have to run additional queries for both validations and + # callbacks, putting a lot of pressure on the database. + # + # To work around this we're using bulk_insert with a single row. This + # allows us to efficiently insert data (even if it's just 1 row) + # without having to use all sorts of hacks to disable callbacks. + Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the merge request this note belongs to. + def find_merge_request_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb new file mode 100644 index 00000000000..966f12c5c2f --- /dev/null +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class DiffNotesImporter + include ParallelScheduling + + def representation_class + Representation::DiffNote + end + + def importer_class + DiffNoteImporter + end + + def sidekiq_worker_class + ImportDiffNoteWorker + end + + def collection_method + :pull_requests_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb new file mode 100644 index 00000000000..bad064b76c8 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_and_label_links_importer.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueAndLabelLinksImporter + attr_reader :issue, :project, :client + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + end + + def execute + IssueImporter.import_if_issue(issue, project, client) + LabelLinksImporter.new(issue, project, client).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb new file mode 100644 index 00000000000..31fefebf787 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueImporter + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, + :issuable_finder + + # Imports an issue if it's a regular issue and not a pull request. + def self.import_if_issue(issue, project, client) + new(issue, project, client).execute unless issue.pull_request? + end + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = GithubImport::IssuableFinder.new(project, issue) + end + + def execute + Issue.transaction do + if (issue_id = create_issue) + create_assignees(issue_id) + issuable_finder.cache_database_id(issue_id) + end + end + end + + # Creates a new GitLab issue for the current GitHub issue. + # + # Returns the ID of the created issue as an Integer. If the issue + # couldn't be created this method will return `nil` instead. + def create_issue + author_id, author_found = user_finder.author_id_for(issue) + + description = + MarkdownText.format(issue.description, issue.author, author_found) + + attributes = { + iid: issue.iid, + title: issue.truncated_title, + author_id: author_id, + project_id: project.id, + description: description, + milestone_id: milestone_finder.id_for(issue), + state: issue.state, + created_at: issue.created_at, + updated_at: issue.updated_at + } + + GithubImport.insert_and_return_id(attributes, project.issues) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the issue. + end + + # Stores all issue assignees in the database. + # + # issue_id - The ID of the created issue. + def create_assignees(issue_id) + assignees = [] + + issue.assignees.each do |assignee| + if (user_id = user_finder.user_id_for(assignee)) + assignees << { issue_id: issue_id, user_id: user_id } + end + end + + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb new file mode 100644 index 00000000000..ac6d0666b3a --- /dev/null +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssuesImporter + include ParallelScheduling + + def importer_class + IssueAndLabelLinksImporter + end + + def representation_class + Representation::Issue + end + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_imported_cache(issue) + issue.number + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb new file mode 100644 index 00000000000..2001b7e3482 --- /dev/null +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelLinksImporter + attr_reader :issue, :project, :client, :label_finder + + # issue - An instance of `Gitlab::GithubImport::Representation::Issue` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(issue, project, client) + @issue = issue + @project = project + @client = client + @label_finder = LabelFinder.new(project) + end + + def execute + create_labels + end + + def create_labels + time = Time.zone.now + rows = [] + target_id = find_target_id + + issue.label_names.each do |label_name| + # Although unlikely it's technically possible for an issue to be + # given a label that was created and assigned after we imported all + # the project's labels. + next unless (label_id = label_finder.id_for(label_name)) + + rows << { + label_id: label_id, + target_id: target_id, + target_type: issue.issuable_type, + created_at: time, + updated_at: time + } + end + + Gitlab::Database.bulk_insert(LabelLink.table_name, rows) + end + + def find_target_id + GithubImport::IssuableFinder.new(project, issue).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb new file mode 100644 index 00000000000..a73033d35ba --- /dev/null +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LabelsImporter + include BulkImporting + + attr_reader :project, :client, :existing_labels + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @client = client + @existing_labels = project.labels.pluck(:title).to_set + end + + def execute + bulk_insert(Label, build_labels) + build_labels_cache + end + + def build_labels + build_database_rows(each_label) + end + + def already_imported?(label) + existing_labels.include?(label.name) + end + + def build_labels_cache + LabelFinder.new(project).build_cache + end + + def build(label) + time = Time.zone.now + + { + title: label.name, + color: '#' + label.color, + project_id: project.id, + type: 'ProjectLabel', + created_at: time, + updated_at: time + } + end + + def each_label + client.labels(project.import_source) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb new file mode 100644 index 00000000000..c53480e828a --- /dev/null +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class MilestonesImporter + include BulkImporting + + attr_reader :project, :client, :existing_milestones + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_milestones = project.milestones.pluck(:iid).to_set + end + + def execute + bulk_insert(Milestone, build_milestones) + build_milestones_cache + end + + def build_milestones + build_database_rows(each_milestone) + end + + def already_imported?(milestone) + existing_milestones.include?(milestone.number) + end + + def build_milestones_cache + MilestoneFinder.new(project).build_cache + end + + def build(milestone) + { + iid: milestone.number, + title: milestone.title, + description: milestone.description, + project_id: project.id, + state: state_for(milestone), + created_at: milestone.created_at, + updated_at: milestone.updated_at + } + end + + def state_for(milestone) + milestone.state == 'open' ? :active : :closed + end + + def each_milestone + client.milestones(project.import_source, state: 'all') + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb new file mode 100644 index 00000000000..c890f2df360 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteImporter + attr_reader :note, :project, :client, :user_finder + + # note - An instance of `Gitlab::GithubImport::Representation::Note`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note, project, client) + @note = note + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + return unless (noteable_id = find_noteable_id) + + author_id, author_found = user_finder.author_id_for(note) + + note_body = + MarkdownText.format(note.note, note.author, author_found) + + attributes = { + noteable_type: note.noteable_type, + noteable_id: noteable_id, + project_id: project.id, + author_id: author_id, + note: note_body, + system: false, + created_at: note.created_at, + updated_at: note.updated_at + } + + # We're using bulk_insert here so we can bypass any validations and + # callbacks. Running these would result in a lot of unnecessary SQL + # queries being executed when importing large projects. + Gitlab::Database.bulk_insert(Note.table_name, [attributes]) + rescue ActiveRecord::InvalidForeignKey + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note. + end + + # Returns the ID of the issue or merge request to create the note for. + def find_noteable_id + GithubImport::IssuableFinder.new(project, note).database_id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb new file mode 100644 index 00000000000..5aec760ea5f --- /dev/null +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NotesImporter + include ParallelScheduling + + def importer_class + NoteImporter + end + + def representation_class + Representation::Note + end + + def sidekiq_worker_class + ImportNoteWorker + end + + def collection_method + :issues_comments + end + + def id_for_already_imported_cache(note) + note.id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb new file mode 100644 index 00000000000..49d859f9624 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestImporter + attr_reader :pull_request, :project, :client, :user_finder, + :milestone_finder, :issuable_finder + + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(pull_request, project, client) + @pull_request = pull_request + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + @milestone_finder = MilestoneFinder.new(project) + @issuable_finder = + GithubImport::IssuableFinder.new(project, pull_request) + end + + def execute + if (mr_id = create_merge_request) + issuable_finder.cache_database_id(mr_id) + end + end + + # Creates the merge request and returns its ID. + # + # This method will return `nil` if the merge request could not be + # created. + def create_merge_request + author_id, author_found = user_finder.author_id_for(pull_request) + + description = MarkdownText + .format(pull_request.description, pull_request.author, author_found) + + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = GithubImport + .insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.find(merge_request_id) + + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_branch_sha = pull_request.target_branch_sha + + merge_request.keep_around_commit + merge_request.merge_request_diffs.create + + merge_request.id + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb new file mode 100644 index 00000000000..5437e32e9f1 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class PullRequestsImporter + include ParallelScheduling + + def importer_class + PullRequestImporter + end + + def representation_class + Representation::PullRequest + end + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def id_for_already_imported_cache(pr) + pr.number + end + + def each_object_to_import + super do |pr| + update_repository if update_repository?(pr) + yield pr + end + end + + def update_repository + # We set this column _before_ fetching the repository, and this is + # deliberate. If we were to update this column after the fetch we may + # miss out on changes pushed during the fetch or between the fetch and + # updating the timestamp. + project.update_column(:last_repository_updated_at, Time.zone.now) + + project.repository.fetch_remote('github', forced: false) + + pname = project.path_with_namespace + + Rails.logger + .info("GitHub importer finished updating repository for #{pname}") + + repository_updates_counter.increment(project: pname) + end + + def update_repository?(pr) + last_update = project.last_repository_updated_at || project.created_at + + return false if pr.updated_at < last_update + + # PRs may be updated without there actually being new commits, thus we + # check to make sure we only re-fetch if truly necessary. + !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + end + + def commit_exists?(sha) + project.repository.lookup(sha) + true + rescue Rugged::Error + false + end + + def collection_method + :pull_requests + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + def repository_updates_counter + @repository_updates_counter ||= Gitlab::Metrics.counter( + :github_importer_repository_updates, + 'The number of times repositories have to be updated again' + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb new file mode 100644 index 00000000000..100f459fdcc --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesImporter + include BulkImporting + + attr_reader :project, :client, :existing_tags + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + @existing_tags = project.releases.pluck(:tag).to_set + end + + def execute + bulk_insert(Release, build_releases) + end + + def build_releases + build_database_rows(each_release) + end + + def already_imported?(release) + existing_tags.include?(release.tag_name) + end + + def build(release) + { + tag: release.tag_name, + description: description_for(release), + created_at: release.created_at, + updated_at: release.updated_at, + project_id: project.id + } + end + + def each_release + client.releases(project.import_source) + end + + def description_for(release) + if release.body.present? + release.body + else + "Release for tag #{release.tag_name}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb new file mode 100644 index 00000000000..0b67fc8db73 --- /dev/null +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class RepositoryImporter + include Gitlab::ShellAdapter + + attr_reader :project, :client + + def initialize(project, client) + @project = project + @client = client + end + + # Returns true if we should import the wiki for the project. + def import_wiki? + client.repository(project.import_source)&.has_wiki && + !project.wiki_repository_exists? + end + + # Imports the repository data. + # + # This method will return true if the data was imported successfully or + # the repository had already been imported before. + def execute + imported = + # It's possible a repository has already been imported when running + # this code, e.g. because we had to retry this job after + # `import_wiki?` raised a rate limit error. In this case we'll skip + # re-importing the main repository. + if project.repository.empty_repo? + import_repository + else + true + end + + update_clone_time if imported + + imported = import_wiki_repository if import_wiki? && imported + + imported + end + + def import_repository + project.ensure_repository + + configure_repository_remote + + project.repository.fetch_remote('github', forced: true) + + true + rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + fail_import("Failed to import the repository: #{e.message}") + end + + def configure_repository_remote + return if project.repository.remote_exists?('github') + + project.repository.add_remote('github', project.import_url) + project.repository.set_import_remote_as_mirror('github') + + project.repository.add_remote_fetch_config( + 'github', + '+refs/pull/*/head:refs/merge-requests/*/head' + ) + end + + def import_wiki_repository + wiki_path = "#{project.disk_path}.wiki" + wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') + storage_path = project.repository_storage_path + + gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + + true + rescue Gitlab::Shell::Error => e + if e.message !~ /repository not exported/ + fail_import("Failed to import the wiki: #{e.message}") + else + true + end + end + + def update_clone_time + project.update_column(:last_repository_updated_at, Time.zone.now) + end + + def fail_import(message) + project.mark_import_as_failed(message) + false + end + end + end + end +end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb new file mode 100644 index 00000000000..211915f1d87 --- /dev/null +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # IssuableFinder can be used for caching and retrieving database IDs for + # issuable objects such as issues and pull requests. By caching these IDs we + # remove the need for running a lot of database queries when importing + # GitHub projects. + class IssuableFinder + attr_reader :project, :object + + # The base cache key to use for storing/retrieving issuable IDs. + CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'.freeze + + # project - An instance of `Project`. + # object - The object to look up or set a database ID for. + def initialize(project, object) + @project = project + @object = object + end + + # Returns the database ID for the object. + # + # This method will return `nil` if no ID could be found. + def database_id + val = Caching.read(cache_key) + + val.to_i if val.present? + end + + # Associates the given database ID with the current object. + # + # database_id - The ID of the corresponding database row. + def cache_database_id(database_id) + Caching.write(cache_key, database_id) + end + + private + + def cache_key + CACHE_KEY % { + project: project.id, + type: cache_key_type, + iid: cache_key_iid + } + end + + # Returns the identifier to use for cache keys. + # + # For issues and pull requests this will be "Issue" or "MergeRequest" + # respectively. For diff notes this will return "MergeRequest", for + # regular notes it will either return "Issue" or "MergeRequest" depending + # on what type of object the note belongs to. + def cache_key_type + if object.respond_to?(:issuable_type) + object.issuable_type + elsif object.respond_to?(:noteable_type) + object.noteable_type + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + + def cache_key_iid + if object.respond_to?(:noteable_id) + object.noteable_id + elsif object.respond_to?(:iid) + object.iid + else + raise( + TypeError, + "Instances of #{object.class} are not supported" + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb new file mode 100644 index 00000000000..9be071141db --- /dev/null +++ b/lib/gitlab/github_import/label_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class LabelFinder + attr_reader :project + + # The base cache key to use for storing/retrieving label IDs. + CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'.freeze + + # project - An instance of `Project`. + def initialize(project) + @project = project + end + + # Returns the label ID for the given name. + def id_for(name) + Caching.read_integer(cache_key_for(name)) + end + + def build_cache + mapping = @project + .labels + .pluck(:id, :name) + .each_with_object({}) do |(id, name), hash| + hash[cache_key_for(name)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(name) + CACHE_KEY % { project: project.id, name: name } + end + end + end +end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb new file mode 100644 index 00000000000..b25c4f7becf --- /dev/null +++ b/lib/gitlab/github_import/markdown_text.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MarkdownText + attr_reader :text, :author, :exists + + def self.format(*args) + new(*args).to_s + end + + # text - The Markdown text as a String. + # author - An instance of `Gitlab::GithubImport::Representation::User` + # exists - Boolean that indicates the user exists in the GitLab database. + def initialize(text, author, exists = false) + @text = text + @author = author + @exists = exists + end + + def to_s + if exists + text + else + "*Created by: #{author.login}*\n\n#{text}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb new file mode 100644 index 00000000000..208d15dc144 --- /dev/null +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class MilestoneFinder + attr_reader :project + + # The base cache key to use for storing/retrieving milestone IDs. + CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'.freeze + + # project - An instance of `Project` + def initialize(project) + @project = project + end + + # issuable - An instance of `Gitlab::GithubImport::Representation::Issue` + # or `Gitlab::GithubImport::Representation::PullRequest`. + def id_for(issuable) + return unless issuable.milestone_number + + Caching.read_integer(cache_key_for(issuable.milestone_number)) + end + + def build_cache + mapping = @project + .milestones + .pluck(:id, :iid) + .each_with_object({}) do |(id, iid), hash| + hash[cache_key_for(iid)] = id + end + + Caching.write_multiple(mapping) + end + + def cache_key_for(iid) + CACHE_KEY % { project: project.id, iid: iid } + end + end + end +end diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/github_import/page_counter.rb new file mode 100644 index 00000000000..c3db2d0b469 --- /dev/null +++ b/lib/gitlab/github_import/page_counter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # PageCounter can be used to keep track of the last imported page of a + # collection, allowing workers to resume where they left off in the event of + # an error. + class PageCounter + attr_reader :cache_key + + # The base cache key to use for storing the last page number. + CACHE_KEY = 'github-importer/page-counter/%{project}/%{collection}'.freeze + + def initialize(project, collection) + @cache_key = CACHE_KEY % { project: project.id, collection: collection } + end + + # Sets the page number to the given value. + # + # Returns true if the page number was overwritten, false otherwise. + def set(page) + Caching.write_if_greater(cache_key, page) + end + + # Returns the current value from the cache. + def current + Caching.read_integer(cache_key) || 1 + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb new file mode 100644 index 00000000000..6da11e6ef08 --- /dev/null +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The ParallelImporter schedules the importing of a GitHub project using + # Sidekiq. + class ParallelImporter + attr_reader :project + + def self.async? + true + end + + def self.imports_repository? + true + end + + def initialize(project) + @project = project + end + + def execute + jid = generate_jid + + # The original import JID is the JID of the RepositoryImportWorker job, + # which will be removed once that job completes. Reusing that JID could + # result in StuckImportJobsWorker marking the job as stuck before we get + # to running Stage::ImportRepositoryWorker. + # + # We work around this by setting the JID to a custom generated one, then + # refreshing it in the various stages whenever necessary. + Gitlab::SidekiqStatus + .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + + project.update_column(:import_jid, jid) + + Stage::ImportRepositoryWorker + .perform_async(project.id) + + true + end + + def generate_jid + "github-importer/#{project.id}" + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb new file mode 100644 index 00000000000..d4d1357f5a3 --- /dev/null +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module ParallelScheduling + attr_reader :project, :client, :page_counter, :already_imported_cache_key + + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'github-importer/already-imported/%{project}/%{collection}'.freeze + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + # parallel - When set to true the objects will be imported in parallel. + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @page_counter = PageCounter.new(project, collection_method) + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def parallel? + @parallel + end + + def execute + retval = + if parallel? + parallel_import + else + sequential_import + end + + # Once we have completed all work we can remove our "already exists" + # cache so we don't put too much pressure on Redis. + # + # We don't immediately remove it since it's technically possible for + # other instances of this job to still run, instead we set the + # expiration time to a lower value. This prevents the other jobs from + # still scheduling duplicates while. Since all work has already been + # completed those jobs will just cycle through any remaining pages while + # not scheduling anything. + Caching.expire(already_imported_cache_key, 15.minutes.to_i) + + retval + end + + # Imports all the objects in sequence in the current thread. + def sequential_import + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + importer_class.new(repr, project, client).execute + end + end + + # Imports all objects in parallel by scheduling a Sidekiq job for every + # individual object. + def parallel_import + waiter = JobWaiter.new + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + sidekiq_worker_class + .perform_async(project.id, repr.to_hash, waiter.key) + + waiter.jobs_remaining += 1 + end + + waiter + end + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + # We inject the page number here to make sure that all importers always + # start where they left off. Simply starting over wouldn't work for + # repositories with a lot of data (e.g. tens of thousands of comments). + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, options) do |page| + # Technically it's possible that the same work is performed multiple + # times, as Sidekiq doesn't guarantee there will ever only be one + # instance of a job. In such a scenario it's possible for one job to + # have a lower page number (e.g. 5) compared to another (e.g. 10). In + # this case we skip over all the objects until we have caught up, + # reducing the number of duplicate jobs scheduled by the provided + # block. + next unless page_counter.set(page.number) + + page.objects.each do |object| + next if already_imported?(object) + + yield object + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(object) + end + end + end + + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(object) + id = id_for_already_imported_cache(object) + + Caching.set_includes?(already_imported_cache_key, id) + end + + # Marks the given object as "already imported". + def mark_as_imported(object) + id = id_for_already_imported_cache(object) + + Caching.set_add(already_imported_cache_key, id) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been imported or not. + # + # object - The object we may want to import. + def id_for_already_imported_cache(object) + raise NotImplementedError + end + + # The class used for converting API responses to Hashes when performing + # the import. + def representation_class + raise NotImplementedError + end + + # The class to use for importing objects when importing them sequentially. + def importer_class + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + # Any options to be passed to the method used for retrieving the data to + # import. + def collection_options + {} + end + end + end +end diff --git a/lib/gitlab/github_import/rate_limit_error.rb b/lib/gitlab/github_import/rate_limit_error.rb new file mode 100644 index 00000000000..cc2de909c29 --- /dev/null +++ b/lib/gitlab/github_import/rate_limit_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Error that will be raised when we're about to reach (or have reached) the + # GitHub API's rate limit. + RateLimitError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/github_import/representation.rb b/lib/gitlab/github_import/representation.rb new file mode 100644 index 00000000000..639477ef2a2 --- /dev/null +++ b/lib/gitlab/github_import/representation.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + TIMESTAMP_KEYS = %i[created_at updated_at merged_at].freeze + + # Converts a Hash with String based keys to one that can be used by the + # various Representation classes. + # + # Example: + # + # Representation.symbolize_hash('number' => 10) # => { number: 10 } + def self.symbolize_hash(raw_hash = nil) + hash = raw_hash.deep_symbolize_keys + + TIMESTAMP_KEYS.each do |key| + hash[key] = Time.parse(hash[key]) if hash[key].is_a?(String) + end + + hash + end + end + end +end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb new file mode 100644 index 00000000000..bb7439a0641 --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class DiffNote + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, + :diff_hunk, :author, :note, :created_at, :updated_at, + :github_id + + NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + + # Builds a diff note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_ID_REGEX) + + unless matches + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: 'MergeRequest', + noteable_id: matches[:iid].to_i, + file_path: note.path, + commit_id: note.commit_id, + diff_hunk: note.diff_hunk, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + def line_code + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + + Gitlab::Git + .diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) + end + + # Returns a Hash that can be used to populate `notes.st_diff`, removing + # the need for requesting Git data for every diff note. + def diff_hash + { + diff: diff_hunk, + new_path: file_path, + old_path: file_path, + + # These fields are not displayed for LegacyDiffNote notes, so it + # doesn't really matter what we set them to. + a_mode: '100644', + b_mode: '100644', + new_file: false + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb new file mode 100644 index 00000000000..c3405759631 --- /dev/null +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ExposeAttribute + extend ActiveSupport::Concern + + module ClassMethods + # Defines getter methods for the given attribute names. + # + # Example: + # + # expose_attribute :iid, :title + def expose_attribute(*names) + names.each do |name| + name = name.to_sym + + define_method(name) { attributes[name] } + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb new file mode 100644 index 00000000000..f3071b3e2b3 --- /dev/null +++ b/lib/gitlab/github_import/representation/issue.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Issue + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :milestone_number, + :created_at, :updated_at, :state, :assignees, + :label_names, :author + + # Builds an issue from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the issue + # details. + def self.from_api_response(issue) + user = + if issue.user + Representation::User.from_api_response(issue.user) + end + + hash = { + iid: issue.number, + title: issue.title, + description: issue.body, + milestone_number: issue.milestone&.number, + state: issue.state == 'open' ? :opened : :closed, + assignees: issue.assignees.map do |u| + Representation::User.from_api_response(u) + end, + label_names: issue.labels.map(&:name), + author: user, + created_at: issue.created_at, + updated_at: issue.updated_at, + pull_request: issue.pull_request ? true : false + } + + new(hash) + end + + # Builds a new issue using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:assignees].map! { |u| Representation::User.from_json_hash(u) } + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A hash containing the raw issue details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + def labels? + label_names && label_names.any? + end + + def pull_request? + attributes[:pull_request] + end + + def issuable_type + pull_request? ? 'MergeRequest' : 'Issue' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb new file mode 100644 index 00000000000..a68bc4c002f --- /dev/null +++ b/lib/gitlab/github_import/representation/note.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Note + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :noteable_id, :noteable_type, :author, :note, + :created_at, :updated_at, :github_id + + NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + + # Builds a note from a GitHub API response. + # + # note - An instance of `Sawyer::Resource` containing the note details. + def self.from_api_response(note) + matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + + if !matches || !matches[:type] + raise( + ArgumentError, + "The note URL #{note.html_url.inspect} is not supported" + ) + end + + noteable_type = + if matches[:type] == 'pull' + 'MergeRequest' + else + 'Issue' + end + + user = Representation::User.from_api_response(note.user) if note.user + hash = { + noteable_type: noteable_type, + noteable_id: matches[:iid].to_i, + author: user, + note: note.body, + created_at: note.created_at, + updated_at: note.updated_at, + github_id: note.id + } + + new(hash) + end + + # Builds a new note using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + new(hash) + end + + # attributes - A Hash containing the raw note details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + + alias_method :issuable_type, :noteable_type + end + end + end +end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb new file mode 100644 index 00000000000..593b491a837 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class PullRequest + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :iid, :title, :description, :source_branch, + :source_branch_sha, :target_branch, :target_branch_sha, + :milestone_number, :author, :assignee, :created_at, + :updated_at, :merged_at, :source_repository_id, + :target_repository_id, :source_repository_owner + + # Builds a PR from a GitHub API response. + # + # issue - An instance of `Sawyer::Resource` containing the PR details. + def self.from_api_response(pr) + assignee = + if pr.assignee + Representation::User.from_api_response(pr.assignee) + end + + user = Representation::User.from_api_response(pr.user) if pr.user + hash = { + iid: pr.number, + title: pr.title, + description: pr.body, + source_branch: pr.head.ref, + target_branch: pr.base.ref, + source_branch_sha: pr.head.sha, + target_branch_sha: pr.base.sha, + source_repository_id: pr.head&.repo&.id, + target_repository_id: pr.base&.repo&.id, + source_repository_owner: pr.head&.user&.login, + state: pr.state == 'open' ? :opened : :closed, + milestone_number: pr.milestone&.number, + author: user, + assignee: assignee, + created_at: pr.created_at, + updated_at: pr.updated_at, + merged_at: pr.merged_at + } + + new(hash) + end + + # Builds a new PR using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + + hash[:state] = hash[:state].to_sym + hash[:author] &&= Representation::User.from_json_hash(hash[:author]) + + # Assignees are optional so we only convert it from a Hash if one was + # set. + hash[:assignee] &&= Representation::User + .from_json_hash(hash[:assignee]) + + new(hash) + end + + # attributes - A Hash containing the raw PR details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def truncated_title + title.truncate(255) + end + + # Returns a formatted source branch. + # + # For cross-project pull requests the branch name will be in the format + # `owner-name:branch-name`. + def formatted_source_branch + if cross_project? && source_repository_owner + "#{source_repository_owner}:#{source_branch}" + elsif source_branch == target_branch + # Sometimes the source and target branch are the same, but GitLab + # doesn't support this. This can happen when both the user and + # source repository have been deleted, and the PR was submitted from + # the fork's master branch. + "#{source_branch}-#{iid}" + else + source_branch + end + end + + def state + if merged_at + :merged + else + attributes[:state] + end + end + + def cross_project? + return true unless source_repository_id + + source_repository_id != target_repository_id + end + + def issuable_type + 'MergeRequest' + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb new file mode 100644 index 00000000000..4a0f36ab8f0 --- /dev/null +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module ToHash + # Converts the current representation to a Hash. The keys of this Hash + # will be Symbols. + def to_hash + hash = {} + + attributes.each do |key, value| + hash[key] = convert_value_for_to_hash(value) + end + + hash + end + + def convert_value_for_to_hash(value) + if value.is_a?(Array) + value.map { |v| convert_value_for_to_hash(v) } + elsif value.respond_to?(:to_hash) + value.to_hash + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb new file mode 100644 index 00000000000..e00dcfca33d --- /dev/null +++ b/lib/gitlab/github_import/representation/user.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class User + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login + + # Builds a user from a GitHub API response. + # + # user - An instance of `Sawyer::Resource` containing the user details. + def self.from_api_response(user) + new(id: user.id, login: user.login) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb new file mode 100644 index 00000000000..4f7324536a0 --- /dev/null +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # The SequentialImporter imports a GitHub project in a single thread, + # without using Sidekiq. This makes it useful for testing purposes as well + # as Rake tasks, but it should be avoided for anything else in favour of the + # parallel importer. + class SequentialImporter + attr_reader :project, :client + + SEQUENTIAL_IMPORTERS = [ + Importer::LabelsImporter, + Importer::MilestonesImporter, + Importer::ReleasesImporter + ].freeze + + PARALLEL_IMPORTERS = [ + Importer::PullRequestsImporter, + Importer::IssuesImporter, + Importer::DiffNotesImporter, + Importer::NotesImporter + ].freeze + + # project - The project to import the data into. + # token - The token to use for the GitHub API. + def initialize(project, token: nil) + @project = project + @client = GithubImport + .new_client_for(project, token: token, parallel: false) + end + + def execute + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + PARALLEL_IMPORTERS.each do |klass| + klass.new(project, client, parallel: false).execute + end + + project.repository.after_import + + true + end + end + end +end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb new file mode 100644 index 00000000000..be1259662a7 --- /dev/null +++ b/lib/gitlab/github_import/user_finder.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Class that can be used for finding a GitLab user ID based on a GitHub user + # ID or username. + # + # Any found user IDs are cached in Redis to reduce the number of SQL queries + # executed over time. Valid keys are refreshed upon access so frequently + # used keys stick around. + # + # Lookups are cached even if no ID was found to remove the need for querying + # the database when most queries are not going to return results anyway. + class UserFinder + attr_reader :project, :client + + # The base cache key to use for caching user IDs for a given GitHub user + # ID. + ID_CACHE_KEY = 'github-import/user-finder/user-id/%s'.freeze + + # The base cache key to use for caching user IDs for a given GitHub email + # address. + ID_FOR_EMAIL_CACHE_KEY = + 'github-import/user-finder/id-for-email/%s'.freeze + + # The base cache key to use for caching the Email addresses of GitHub + # usernames. + EMAIL_FOR_USERNAME_CACHE_KEY = + 'github-import/user-finder/email-for-username/%s'.freeze + + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(project, client) + @project = project + @client = client + end + + # Returns the GitLab user ID of an object's author. + # + # If the object has no author ID we'll use the ID of the GitLab ghost + # user. + def author_id_for(object) + id = + if object&.author + user_id_for(object.author) + else + GithubImport.ghost_user_id + end + + if id + [id, true] + else + [project.creator_id, false] + end + end + + # Returns the GitLab user ID of an issuable's assignee. + def assignee_id_for(issuable) + user_id_for(issuable.assignee) if issuable.assignee + end + + # Returns the GitLab user ID for a GitHub user. + # + # user - An instance of `Gitlab::GithubImport::Representation::User`. + def user_id_for(user) + find(user.id, user.login) + end + + # Returns the GitLab ID for the given GitHub ID or username. + # + # id - The ID of the GitHub user. + # username - The username of the GitHub user. + def find(id, username) + email = email_for_github_username(username) + cached, found_id = find_from_cache(id, email) + + return found_id if found_id + + # We only want to query the database if necessary. If previous lookups + # didn't yield a user ID we won't query the database again until the + # keys expire. + find_id_from_database(id, email) unless cached + end + + # Finds a user ID from the cache for a given GitHub ID or Email. + def find_from_cache(id, email = nil) + id_exists, id_for_github_id = cached_id_for_github_id(id) + + return [id_exists, id_for_github_id] if id_for_github_id + + # Just in case no Email address could be retrieved (for whatever reason) + return [false] unless email + + cached_id_for_github_email(email) + end + + # Finds a GitLab user ID from the database for a given GitHub user ID or + # Email. + def find_id_from_database(id, email) + id_for_github_id(id) || id_for_github_email(email) + end + + def email_for_github_username(username) + cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username + email = Caching.read(cache_key) + + unless email + user = client.user(username) + email = Caching.write(cache_key, user.email) if user + end + + email + end + + def cached_id_for_github_id(id) + read_id_from_cache(ID_CACHE_KEY % id) + end + + def cached_id_for_github_email(email) + read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email) + end + + # Queries and caches the GitLab user ID for a GitHub user ID, if one was + # found. + def id_for_github_id(id) + gitlab_id = query_id_for_github_id(id) || nil + + Caching.write(ID_CACHE_KEY % id, gitlab_id) + end + + # Queries and caches the GitLab user ID for a GitHub email, if one was + # found. + def id_for_github_email(email) + gitlab_id = query_id_for_github_email(email) || nil + + Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) + end + + def query_id_for_github_id(id) + User.for_github_id(id).pluck(:id).first + end + + def query_id_for_github_email(email) + User.by_any_email(email).pluck(:id).first + end + + # Reads an ID from the cache. + # + # The return value is an Array with two values: + # + # 1. A boolean indicating if the key was present or not. + # 2. The ID as an Integer, or nil in case no ID could be found. + def read_id_from_cache(key) + value = Caching.read(key) + exists = !value.nil? + number = value.to_i + + # The cache key may be empty to indicate a previously looked up user for + # which we couldn't find an ID. + [exists, number.positive? ? number : nil] + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3a666c2268b..dfcdfc307b6 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,7 +20,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') - gon.sprite_icons = ActionController::Base.helpers.asset_path('icons.svg') + gon.sprite_icons = IconsHelper.sprite_icon_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index de9cab80a02..e29dd0d5b0e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -4,7 +4,6 @@ module Gitlab SAFE_HOOK_ATTRIBUTES = %i[ assignee_id author_id - branch_name closed_at confidential created_at @@ -29,6 +28,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignees labels + total_time_spent ].freeze attr_accessor :issue diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index eaef19c9d04..ae9b68eb648 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -19,7 +19,6 @@ module Gitlab merge_user_id merge_when_pipeline_succeeds milestone_id - ref_fetched source_branch source_project_id state @@ -34,6 +33,7 @@ module Gitlab SAFE_HOOK_RELATIONS = %i[ assignee labels + total_time_spent ].freeze attr_accessor :merge_request diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 561779182bc..263599831bf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -54,7 +54,6 @@ project_tree: - :auto_devops - :triggers - :pipeline_schedules - - :cluster - :services - :hooks - protected_branches: @@ -63,6 +62,7 @@ project_tree: - protected_tags: - :create_access_levels - :project_feature + - :custom_attributes # Only include the following attributes for the models specified. included_attributes: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index fbdd74788bc..c14646b0611 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,10 @@ module Gitlab module ImportExport class Importer + def self.imports_repository? + true + end + def initialize(project) @archive_file = project.import_source @current_user = project.creator diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 81a213e8321..61db4bd9ccc 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -26,7 +26,7 @@ module Gitlab end def fetch_ref - @project.repository.fetch_ref(@project.repository.path, @diff_head_sha, @merge_request.source_branch) + @project.repository.fetch_ref(@project.repository, source_ref: @diff_head_sha, target_ref: @merge_request.source_branch) end def branch_exists?(branch_name) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 469b230377d..2b34ceb5831 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -8,8 +8,6 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', - cluster: 'Gcp::Cluster', - clusters: 'Gcp::Cluster', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -17,7 +15,8 @@ module Gitlab labels: :project_labels, priorities: :label_priorities, auto_devops: :project_auto_devops, - label: :project_label }.freeze + label: :project_label, + custom_attributes: 'ProjectCustomAttribute' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5404dc11a87..eeb03625479 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -8,14 +8,14 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ - ImportSource.new('github', 'GitHub', Github::Import), + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repo by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) ].freeze class << self diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 977c05910d3..0c9de72329c 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -1,6 +1,14 @@ module Gitlab module IssuableMetadata def issuable_meta_data(issuable_collection, collection_type) + # ActiveRecord uses Object#extend for null relations. + if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && + issuable_collection.respond_to?(:limit_value) && + issuable_collection.limit_value.nil? + + raise 'Collection must have a limit applied for preloading meta-data' + end + # map has to be used here since using pluck or select will # throw an error when ordering issuables by priority which inserts # a new order into the collection. diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 4d6bbda15f3..f654508c391 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,11 +19,13 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } end - attr_reader :key, :jobs_remaining, :finished + attr_reader :key, :finished + attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for - def initialize(jobs_remaining) - @key = "gitlab:job_waiter:#{SecureRandom.uuid}" + # key - The key of this waiter. + def initialize(jobs_remaining = 0, key = "gitlab:job_waiter:#{SecureRandom.uuid}") + @key = key @jobs_remaining = jobs_remaining @finished = [] end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb new file mode 100644 index 00000000000..7a50f07f3c5 --- /dev/null +++ b/lib/gitlab/kubernetes/helm.rb @@ -0,0 +1,96 @@ +module Gitlab + module Kubernetes + class Helm + HELM_VERSION = '2.7.0'.freeze + NAMESPACE = 'gitlab-managed-apps'.freeze + INSTALL_DEPS = <<-EOS.freeze + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + EOS + + InstallCommand = Struct.new(:name, :install_helm, :chart) do + def pod_name + "install-#{name}" + end + end + + def initialize(kubeclient) + @kubeclient = kubeclient + @namespace = Namespace.new(NAMESPACE, kubeclient) + end + + def install(command) + @namespace.ensure_exists! + @kubeclient.create_pod(pod_resource(command)) + end + + ## + # Returns Pod phase + # + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + # + # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" + # + def installation_status(pod_name) + @kubeclient.get_pod(pod_name, @namespace.name).status.phase + end + + def installation_log(pod_name) + @kubeclient.get_pod_log(pod_name, @namespace.name).body + end + + def delete_installation_pod!(pod_name) + @kubeclient.delete_pod(pod_name, @namespace.name) + end + + private + + def pod_resource(command) + labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } + metadata = { name: command.pod_name, namespace: @namespace.name, labels: labels } + container = { + name: 'helm', + image: 'alpine:3.6', + env: generate_pod_env(command), + command: %w(/bin/sh), + args: %w(-c $(COMMAND_SCRIPT)) + } + spec = { containers: [container], restartPolicy: 'Never' } + + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) + end + + def generate_pod_env(command) + { + HELM_VERSION: HELM_VERSION, + TILLER_NAMESPACE: @namespace.name, + COMMAND_SCRIPT: generate_script(command) + }.map { |key, value| { name: key, value: value } } + end + + def generate_script(command) + [ + INSTALL_DEPS, + helm_init_command(command), + helm_install_command(command) + ].join("\n") + end + + def helm_init_command(command) + if command.install_helm + 'helm init >/dev/null' + else + 'helm init --client-only >/dev/null' + end + end + + def helm_install_command(command) + return if command.chart.nil? + + "helm install #{command.chart} --name #{command.name} --namespace #{@namespace.name} >/dev/null" + end + end + end +end diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb new file mode 100644 index 00000000000..c8479fbc0e8 --- /dev/null +++ b/lib/gitlab/kubernetes/namespace.rb @@ -0,0 +1,29 @@ +module Gitlab + module Kubernetes + class Namespace + attr_accessor :name + + def initialize(name, client) + @name = name + @client = client + end + + def exists? + @client.get_namespace(name) + rescue ::KubeException => ke + raise ke unless ke.error_code == 404 + false + end + + def create! + resource = ::Kubeclient::Resource.new(metadata: { name: name }) + + @client.create_namespace(resource) + end + + def ensure_exists! + exists? || create! + end + end + end +end diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb new file mode 100644 index 00000000000..f3842cdf762 --- /dev/null +++ b/lib/gitlab/kubernetes/pod.rb @@ -0,0 +1,12 @@ +module Gitlab + module Kubernetes + module Pod + PENDING = 'Pending'.freeze + RUNNING = 'Running'.freeze + SUCCEEDED = 'Succeeded'.freeze + FAILED = 'Failed'.freeze + UNKNOWN = 'Unknown'.freeze + PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index f330041cc00..2f07fde406c 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BaseFormatter attr_reader :client, :formatter, :project, :raw_data diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 8aa885fb811..80fe1d67209 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class BranchFormatter < BaseFormatter delegate :repo, :sha, :ref, to: :raw_data diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb new file mode 100644 index 00000000000..53c910d44bd --- /dev/null +++ b/lib/gitlab/legacy_github_import/client.rb @@ -0,0 +1,148 @@ +module Gitlab + module LegacyGithubImport + class Client + GITHUB_SAFE_REMAINING_REQUESTS = 100 + GITHUB_SAFE_SLEEP_TIME = 500 + + attr_reader :access_token, :host, :api_version + + def initialize(access_token, host: nil, api_version: 'v3') + @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version + @users = {} + + if access_token + ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: api_endpoint, + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end + + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' + end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) + end + + def authorize_url(redirect_uri) + client.auth_code.authorize_url({ + redirect_uri: redirect_uri, + scope: "repo, user, user:email" + }) + end + + def get_token(code) + client.auth_code.get_token(code).token + end + + def method_missing(method, *args, &block) + if api.respond_to?(method) + request(method, *args, &block) + else + super(method, *args, &block) + end + end + + def respond_to?(method) + api.respond_to?(method) || super + end + + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + + private + + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + + def config + Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } + end + + def github_options + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end + end + + def rate_limit + api.rate_limit! + # GitHub Rate Limit API returns 404 when the rate limit is + # disabled. In this case we just want to return gracefully + # instead of spitting out an error. + rescue Octokit::NotFound + nil + end + + def has_rate_limit? + return @has_rate_limit if defined?(@has_rate_limit) + + @has_rate_limit = rate_limit.present? + end + + def rate_limit_exceed? + has_rate_limit? && rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME + end + + def request(method, *args, &block) + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = api.__send__(method, *args) # rubocop:disable GitlabSecurity/PublicSend + return data unless data.is_a?(Array) + + last_response = api.last_response + + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + yield last_response.data if last_response.data.is_a?(Array) + end + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index 8911b81ec9a..d2c7a8ae9f4 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class CommentFormatter < BaseFormatter attr_writer :author_id diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b8c07460ebb..12c968805f5 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class Importer include Gitlab::ShellAdapter diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index 27b171d6ddb..de55382d3ad 100644 --- a/lib/gitlab/github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssuableFormatter < BaseFormatter attr_writer :assignee_id, :author_id diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 977cd0423ba..4c8825ccf19 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class IssueFormatter < IssuableFormatter def attributes { diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 211ccdc51bb..c3eed12e739 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class LabelFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index dd782eff059..a565294384d 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class MilestoneFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index a55adc9b1c8..41e7eac4d08 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ProjectCreator include Gitlab::CurrentSettings diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 150afa31432..94c2e99066a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class PullRequestFormatter < IssuableFormatter delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 1ad702a6058..3ed9d4f76da 100644 --- a/lib/gitlab/github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class ReleaseFormatter < BaseFormatter def attributes { diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 04c2964da20..6d8055622f1 100644 --- a/lib/gitlab/github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class UserFormatter attr_reader :client, :raw diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index ca8d96f5650..27f45875c7c 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -1,5 +1,5 @@ module Gitlab - module GithubImport + module LegacyGithubImport class WikiFormatter attr_reader :project diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 8e57ba831c5..ead5d566871 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -27,6 +27,10 @@ module Gitlab end end + def deploy_key_pushable?(project) + actor.is_a?(DeployKey) && actor.can_push_to?(project) + end + def user? actor.is_a?(User) end diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb new file mode 100644 index 00000000000..5919ebb1493 --- /dev/null +++ b/lib/gitlab/metrics/background_transaction.rb @@ -0,0 +1,14 @@ +module Gitlab + module Metrics + class BackgroundTransaction < Transaction + def initialize(worker_class) + super() + @worker_class = worker_class + end + + def labels + { controller: @worker_class.name, action: 'perform' } + end + end + end +end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb deleted file mode 100644 index 716d20bb91a..00000000000 --- a/lib/gitlab/metrics/base_sampler.rb +++ /dev/null @@ -1,63 +0,0 @@ -require 'logger' -module Gitlab - module Metrics - class BaseSampler < Daemon - # interval - The sampling interval in seconds. - def initialize(interval) - interval_half = interval.to_f / 2 - - @interval = interval - @interval_steps = (-interval_half..interval_half).step(0.1).to_a - - super() - end - - def safe_sample - sample - rescue => e - Rails.logger.warn("#{self.class}: #{e}, stopping") - stop - end - - def sample - raise NotImplementedError - end - - # Returns the sleep interval with a random adjustment. - # - # The random adjustment is put in place to ensure we: - # - # 1. Don't generate samples at the exact same interval every time (thus - # potentially missing anything that happens in between samples). - # 2. Don't sample data at the same interval two times in a row. - def sleep_interval - while (step = @interval_steps.sample) - if step != @last_step - @last_step = step - - return @interval + @last_step - end - end - end - - private - - attr_reader :running - - def start_working - @running = true - sleep(sleep_interval) - - while running - safe_sample - - sleep(sleep_interval) - end - end - - def stop_working - @running = false - end - end - end -end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index c4dc061eda1..3c5f9099584 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -11,6 +11,8 @@ module Gitlab settings[:enabled] || false end + # Prometheus histogram buckets used for arbitrary code measurements + EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1].freeze RAILS_ROOT = Rails.root.to_s METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ @@ -99,24 +101,27 @@ module Gitlab cpu_stop = System.cpu_time real_stop = Time.now.to_f - real_time = (real_stop - real_start) * 1000.0 + real_time = (real_stop - real_start) cpu_time = cpu_stop - cpu_start - trans.increment("#{name}_real_time", real_time) - trans.increment("#{name}_cpu_time", cpu_time) - trans.increment("#{name}_call_count", 1) + Gitlab::Metrics.histogram("gitlab_#{name}_real_duration_seconds".to_sym, + "Measure #{name}", + Transaction::BASE_LABELS, + EXECUTION_MEASUREMENT_BUCKETS) + .observe(trans.labels, real_time) - retval - end + Gitlab::Metrics.histogram("gitlab_#{name}_cpu_duration_seconds".to_sym, + "Measure #{name}", + Transaction::BASE_LABELS, + EXECUTION_MEASUREMENT_BUCKETS) + .observe(trans.labels, cpu_time / 1000.0) - # Adds a tag to the current transaction (if any) - # - # name - The name of the tag to add. - # value - The value of the tag. - def tag_transaction(name, value) - trans = current_transaction + # InfluxDB stores the _real_time time values as milliseconds + trans.increment("#{name}_real_time", real_time * 1000, false) + trans.increment("#{name}_cpu_time", cpu_time, false) + trans.increment("#{name}_call_count", 1, false) - trans&.add_tag(name, value) + retval end # Sets the action of the current transaction (if any) diff --git a/lib/gitlab/metrics/influx_sampler.rb b/lib/gitlab/metrics/influx_sampler.rb deleted file mode 100644 index 6db1dd755b7..00000000000 --- a/lib/gitlab/metrics/influx_sampler.rb +++ /dev/null @@ -1,101 +0,0 @@ -module Gitlab - module Metrics - # Class that sends certain metrics to InfluxDB at a specific interval. - # - # This class is used to gather statistics that can't be directly associated - # with a transaction such as system memory usage, garbage collection - # statistics, etc. - class InfluxSampler < BaseSampler - # interval - The sampling interval in seconds. - def initialize(interval = Metrics.settings[:sample_interval]) - super(interval) - @last_step = nil - - @metrics = [] - - @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) - @last_major_gc = Delta.new(GC.stat[:major_gc_count]) - - if Gitlab::Metrics.mri? - require 'allocations' - - Allocations.start - end - end - - def sample - sample_memory_usage - sample_file_descriptors - sample_objects - sample_gc - - flush - ensure - GC::Profiler.clear - @metrics.clear - end - - def flush - Metrics.submit_metrics(@metrics.map(&:to_hash)) - end - - def sample_memory_usage - add_metric('memory_usage', value: System.memory_usage) - end - - def sample_file_descriptors - add_metric('file_descriptors', value: System.file_descriptor_count) - end - - if Metrics.mri? - def sample_objects - sample = Allocations.to_hash - counts = sample.each_with_object({}) do |(klass, count), hash| - name = klass.name - - next unless name - - hash[name] = count - end - - # Symbols aren't allocated so we'll need to add those manually. - counts['Symbol'] = Symbol.all_symbols.length - - counts.each do |name, count| - add_metric('object_counts', { count: count }, type: name) - end - end - else - def sample_objects - end - end - - def sample_gc - time = GC::Profiler.total_time * 1000.0 - stats = GC.stat.merge(total_time: time) - - # We want the difference of GC runs compared to the last sample, not the - # total amount since the process started. - stats[:minor_gc_count] = - @last_minor_gc.compared_with(stats[:minor_gc_count]) - - stats[:major_gc_count] = - @last_major_gc.compared_with(stats[:major_gc_count]) - - stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - - add_metric('gc_statistics', stats) - end - - def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' - - @metrics << Metric.new("#{prefix}#{series}", values, tags) - end - - def sidekiq? - Sidekiq.server? - end - end - end -end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 6aa38542cb4..023e9963493 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -118,19 +118,21 @@ module Gitlab def self.instrument(type, mod, name) return unless Metrics.enabled? - name = name.to_sym + name = name.to_sym target = type == :instance ? mod : mod.singleton_class if type == :instance target = mod - label = "#{mod.name}##{name}" + method_name = "##{name}" method = mod.instance_method(name) else target = mod.singleton_class - label = "#{mod.name}.#{name}" + method_name = ".#{name}" method = mod.method(name) end + label = "#{mod.name}#{method_name}" + unless instrumented?(target) target.instance_variable_set(PROXY_IVAR, Module.new) end @@ -153,7 +155,8 @@ module Gitlab proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) if trans = Gitlab::Metrics::Instrumentation.transaction - trans.method_call_for(#{label.to_sym.inspect}).measure { super } + trans.method_call_for(#{label.to_sym.inspect}, #{mod.name.inspect}, "#{method_name}") + .measure { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index d3465e5ec19..90235095306 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -2,15 +2,45 @@ module Gitlab module Metrics # Class for tracking timing information about method calls class MethodCall - attr_reader :real_time, :cpu_time, :call_count + MUTEX = Mutex.new + BASE_LABELS = { module: nil, method: nil }.freeze + attr_reader :real_time, :cpu_time, :call_count, :labels + + def self.call_real_duration_histogram + return @call_real_duration_histogram if @call_real_duration_histogram + + MUTEX.synchronize do + @call_real_duration_histogram ||= Gitlab::Metrics.histogram( + :gitlab_method_call_real_duration_seconds, + 'Method calls real duration', + Transaction::BASE_LABELS.merge(BASE_LABELS), + [0.1, 0.2, 0.5, 1, 2, 5, 10] + ) + end + end + + def self.call_cpu_duration_histogram + return @call_cpu_duration_histogram if @call_cpu_duration_histogram + + MUTEX.synchronize do + @call_duration_histogram ||= Gitlab::Metrics.histogram( + :gitlab_method_call_cpu_duration_seconds, + 'Method calls cpu duration', + Transaction::BASE_LABELS.merge(BASE_LABELS), + [0.1, 0.2, 0.5, 1, 2, 5, 10] + ) + end + end # name - The full name of the method (including namespace) such as # `User#sign_in`. # - # series - The series to use for storing the data. - def initialize(name, series) + def initialize(name, module_name, method_name, transaction) + @module_name = module_name + @method_name = method_name + @transaction = transaction @name = name - @series = series + @labels = { module: @module_name, method: @method_name } @real_time = 0 @cpu_time = 0 @call_count = 0 @@ -22,21 +52,27 @@ module Gitlab start_cpu = System.cpu_time retval = yield - @real_time += System.monotonic_time - start_real - @cpu_time += System.cpu_time - start_cpu + real_time = System.monotonic_time - start_real + cpu_time = System.cpu_time - start_cpu + + @real_time += real_time + @cpu_time += cpu_time @call_count += 1 + self.class.call_real_duration_histogram.observe(@transaction.labels.merge(labels), real_time / 1000.0) + self.class.call_cpu_duration_histogram.observe(@transaction.labels.merge(labels), cpu_time / 1000.0) + retval end # Returns a Metric instance of the current method call. def to_metric Metric.new( - @series, + Instrumentation.series, { - duration: real_time, + duration: real_time, cpu_duration: cpu_time, - call_count: call_count + call_count: call_count }, method: @name ) diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index b5f9dafccab..75461b45005 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -5,6 +5,9 @@ module Gitlab module Prometheus include Gitlab::CurrentSettings + REGISTRY_MUTEX = Mutex.new + PROVIDER_MUTEX = Mutex.new + def metrics_folder_present? multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir @@ -21,23 +24,38 @@ module Gitlab end def registry - @registry ||= ::Prometheus::Client.registry + return @registry if @registry + + REGISTRY_MUTEX.synchronize do + @registry ||= ::Prometheus::Client.registry + end end def counter(name, docstring, base_labels = {}) - provide_metric(name) || registry.counter(name, docstring, base_labels) + safe_provide_metric(:counter, name, docstring, base_labels) end def summary(name, docstring, base_labels = {}) - provide_metric(name) || registry.summary(name, docstring, base_labels) + safe_provide_metric(:summary, name, docstring, base_labels) end def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) - provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode) + safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode) end def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) - provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets) + safe_provide_metric(:histogram, name, docstring, base_labels, buckets) + end + + private + + def safe_provide_metric(method, name, *args) + metric = provide_metric(name) + return metric if metric + + PROVIDER_MUTEX.synchronize do + provide_metric(name) || registry.method(method).call(name, *args) + end end def provide_metric(name) @@ -48,8 +66,6 @@ module Gitlab end end - private - def prometheus_metrics_enabled_unmemoized metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index adc0db1a874..2d45765df3f 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -2,20 +2,6 @@ module Gitlab module Metrics # Rack middleware for tracking Rails and Grape requests. class RackMiddleware - CONTROLLER_KEY = 'action_controller.instance'.freeze - ENDPOINT_KEY = 'api.endpoint'.freeze - CONTENT_TYPES = { - 'text/html' => :html, - 'text/plain' => :txt, - 'application/json' => :json, - 'text/js' => :js, - 'application/atom+xml' => :atom, - 'image/png' => :png, - 'image/jpeg' => :jpeg, - 'image/gif' => :gif, - 'image/svg+xml' => :svg - }.freeze - def initialize(app) @app = app end @@ -35,12 +21,6 @@ module Gitlab # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure - if env[CONTROLLER_KEY] - tag_controller(trans, env) - elsif env[ENDPOINT_KEY] - tag_endpoint(trans, env) - end - trans.finish end @@ -48,60 +28,19 @@ module Gitlab end def transaction_from_env(env) - trans = Transaction.new + trans = WebTransaction.new(env) - trans.set(:request_uri, filtered_path(env)) - trans.set(:request_method, env['REQUEST_METHOD']) + trans.set(:request_uri, filtered_path(env), false) + trans.set(:request_method, env['REQUEST_METHOD'], false) trans end - def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - action = "#{controller.class.name}##{controller.action_name}" - suffix = CONTENT_TYPES[controller.content_type] - - if suffix && suffix != :html - action += ".#{suffix}" - end - - trans.action = action - end - - def tag_endpoint(trans, env) - endpoint = env[ENDPOINT_KEY] - - begin - route = endpoint.route - rescue - # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] - # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response - # so we're rescuing exceptions and bailing out - end - - if route - path = endpoint_paths_cache[route.request_method][route.path] - trans.action = "Grape##{route.request_method} #{path}" - end - end - private def filtered_path(env) ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] end - - def endpoint_paths_cache - @endpoint_paths_cache ||= Hash.new do |hash, http_method| - hash[http_method] = Hash.new do |inner_hash, raw_path| - inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) - end - end - end - - def endpoint_instrumentable_path(raw_path) - raw_path.sub('(.:format)', '').sub('/:version', '') - end end end end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb new file mode 100644 index 00000000000..37f90c4673d --- /dev/null +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -0,0 +1,64 @@ +require 'logger' + +module Gitlab + module Metrics + module Samplers + class BaseSampler < Daemon + # interval - The sampling interval in seconds. + def initialize(interval) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + + super() + end + + def safe_sample + sample + rescue => e + Rails.logger.warn("#{self.class}: #{e}, stopping") + stop + end + + def sample + raise NotImplementedError + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end + + private + + attr_reader :running + + def start_working + @running = true + sleep(sleep_interval) + while running + safe_sample + sleep(sleep_interval) + end + end + + def stop_working + @running = false + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb new file mode 100644 index 00000000000..f4f9b5ca792 --- /dev/null +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -0,0 +1,103 @@ +module Gitlab + module Metrics + module Samplers + # Class that sends certain metrics to InfluxDB at a specific interval. + # + # This class is used to gather statistics that can't be directly associated + # with a transaction such as system memory usage, garbage collection + # statistics, etc. + class InfluxSampler < BaseSampler + # interval - The sampling interval in seconds. + def initialize(interval = Metrics.settings[:sample_interval]) + super(interval) + @last_step = nil + + @metrics = [] + + @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) + @last_major_gc = Delta.new(GC.stat[:major_gc_count]) + + if Gitlab::Metrics.mri? + require 'allocations' + + Allocations.start + end + end + + def sample + sample_memory_usage + sample_file_descriptors + sample_objects + sample_gc + + flush + ensure + GC::Profiler.clear + @metrics.clear + end + + def flush + Metrics.submit_metrics(@metrics.map(&:to_hash)) + end + + def sample_memory_usage + add_metric('memory_usage', value: System.memory_usage) + end + + def sample_file_descriptors + add_metric('file_descriptors', value: System.file_descriptor_count) + end + + if Metrics.mri? + def sample_objects + sample = Allocations.to_hash + counts = sample.each_with_object({}) do |(klass, count), hash| + name = klass.name + + next unless name + + hash[name] = count + end + + # Symbols aren't allocated so we'll need to add those manually. + counts['Symbol'] = Symbol.all_symbols.length + + counts.each do |name, count| + add_metric('object_counts', { count: count }, type: name) + end + end + else + def sample_objects + end + end + + def sample_gc + time = GC::Profiler.total_time * 1000.0 + stats = GC.stat.merge(total_time: time) + + # We want the difference of GC runs compared to the last sample, not the + # total amount since the process started. + stats[:minor_gc_count] = + @last_minor_gc.compared_with(stats[:minor_gc_count]) + + stats[:major_gc_count] = + @last_major_gc.compared_with(stats[:major_gc_count]) + + stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] + + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb new file mode 100644 index 00000000000..8b5a60e6b8b --- /dev/null +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -0,0 +1,110 @@ +require 'prometheus/client/support/unicorn' + +module Gitlab + module Metrics + module Samplers + class RubySampler < BaseSampler + def metrics + @metrics ||= init_metrics + end + + def with_prefix(prefix, name) + "ruby_#{prefix}_#{name}".to_sym + end + + def to_doc_string(name) + name.to_s.humanize + end + + def labels + {} + end + + def initialize(interval) + super(interval) + + if Metrics.mri? + require 'allocations' + + Allocations.start + end + end + + def init_metrics + metrics = {} + metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {}) + metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + GC.stat.keys.each do |key| + metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + end + + metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + + metrics + end + + def sample + start_time = System.monotonic_time + sample_gc + sample_objects + + metrics[:memory_usage].set(labels, System.memory_usage) + metrics[:file_descriptors].set(labels, System.file_descriptor_count) + + metrics[:sampler_duration].observe(labels.merge(worker_label), (System.monotonic_time - start_time) / 1000.0) + ensure + GC::Profiler.clear + end + + private + + def sample_gc + metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) + + GC.stat.each do |key, value| + metrics[key].set(labels, value) + end + end + + def sample_objects + list_objects.each do |name, count| + metrics[:objects_total].set(labels.merge(class: name), count) + end + end + + if Metrics.mri? + def list_objects + sample = Allocations.to_hash + counts = sample.each_with_object({}) do |(klass, count), hash| + name = klass.name + + next unless name + + hash[name] = count + end + + # Symbols aren't allocated so we'll need to add those manually. + counts['Symbol'] = Symbol.all_symbols.length + counts + end + else + def list_objects + end + end + + def worker_label + return {} unless defined?(Unicorn::Worker) + worker_no = ::Prometheus::Client::Support::Unicorn.worker_id + + if worker_no + { unicorn: worker_no } + else + { unicorn: 'master' } + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb new file mode 100644 index 00000000000..ea325651fbb --- /dev/null +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -0,0 +1,50 @@ +module Gitlab + module Metrics + module Samplers + class UnicornSampler < BaseSampler + def initialize(interval) + super(interval) + end + + def unicorn_active_connections + @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + end + + def unicorn_queued_connections + @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + end + + def enabled? + # Raindrops::Linux.tcp_listener_stats is only present on Linux + unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) + end + + def sample + Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + end + + Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + end + end + + private + + def tcp_listeners + @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) + end + + def unix_listeners + @unix_listeners ||= Unicorn.listener_names - tcp_listeners + end + + def unicorn_with_listeners? + defined?(Unicorn) && Unicorn.listener_names.any? + end + end + end + end +end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index b983a40611f..df4bdf16847 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,14 +5,12 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new("#{worker.class.name}#perform") + trans = BackgroundTransaction.new(worker.class) begin # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } - - worker.metrics_tags.each { |tag, value| trans.add_tag(tag, value) } if worker.respond_to?(:metrics_tags) rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index d435a33e9c7..3da474fc1ec 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -15,10 +15,24 @@ module Gitlab private + def metric_view_rendering_duration_seconds + @metric_view_rendering_duration_seconds ||= Gitlab::Metrics.histogram( + :gitlab_view_rendering_duration_seconds, + 'View rendering time', + Transaction::BASE_LABELS.merge({ path: nil }), + [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] + ) + end + def track(event) values = values_for(event) tags = tags_for(event) + metric_view_rendering_duration_seconds.observe( + current_transaction.labels.merge(tags), + event.duration + ) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 96cad941d5c..064299f40c8 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -7,9 +7,10 @@ module Gitlab def sql(event) return unless current_transaction + metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) - current_transaction.increment(:sql_duration, event.duration) - current_transaction.increment(:sql_count, 1) + current_transaction.increment(:sql_duration, event.duration, false) + current_transaction.increment(:sql_count, 1, false) end private @@ -17,6 +18,15 @@ module Gitlab def current_transaction Transaction.current end + + def metric_sql_duration_seconds + @metric_sql_duration_seconds ||= Gitlab::Metrics.histogram( + :gitlab_sql_duration_seconds, + 'SQL time', + Transaction::BASE_LABELS, + [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] + ) + end end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index aaed2184f44..efd3c9daf79 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -7,28 +7,29 @@ module Gitlab attach_to :active_support def cache_read(event) - increment(:cache_read, event.duration) + observe(:read, event.duration) return unless current_transaction return if event.payload[:super_operation] == :fetch if event.payload[:hit] - current_transaction.increment(:cache_read_hit_count, 1) + current_transaction.increment(:cache_read_hit_count, 1, false) else - current_transaction.increment(:cache_read_miss_count, 1) + metric_cache_misses_total.increment(current_transaction.labels) + current_transaction.increment(:cache_read_miss_count, 1, false) end end def cache_write(event) - increment(:cache_write, event.duration) + observe(:write, event.duration) end def cache_delete(event) - increment(:cache_delete, event.duration) + observe(:delete, event.duration) end def cache_exist?(event) - increment(:cache_exists, event.duration) + observe(:exists, event.duration) end def cache_fetch_hit(event) @@ -40,16 +41,18 @@ module Gitlab def cache_generate(event) return unless current_transaction + metric_cache_misses_total.increment(current_transaction.labels) current_transaction.increment(:cache_read_miss_count, 1) end - def increment(key, duration) + def observe(key, duration) return unless current_transaction - current_transaction.increment(:cache_duration, duration) - current_transaction.increment(:cache_count, 1) - current_transaction.increment("#{key}_duration".to_sym, duration) - current_transaction.increment("#{key}_count".to_sym, 1) + metric_cache_operation_duration_seconds.observe(current_transaction.labels.merge({ operation: key }), duration / 1000.0) + current_transaction.increment(:cache_duration, duration, false) + current_transaction.increment(:cache_count, 1, false) + current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) + current_transaction.increment("cache_#{key}_count".to_sym, 1, false) end private @@ -57,6 +60,23 @@ module Gitlab def current_transaction Transaction.current end + + def metric_cache_operation_duration_seconds + @metric_cache_operation_duration_seconds ||= Gitlab::Metrics.histogram( + :gitlab_cache_operation_duration_seconds, + 'Cache access time', + Transaction::BASE_LABELS.merge({ action: nil }), + [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] + ) + end + + def metric_cache_misses_total + @metric_cache_misses_total ||= Gitlab::Metrics.counter( + :gitlab_cache_misses_total, + 'Cache read miss', + Transaction::BASE_LABELS + ) + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 4f9fb1c7853..ee3afc5ffdb 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -2,34 +2,33 @@ module Gitlab module Metrics # Class for storing metrics information of a single transaction. class Transaction + # base labels shared among all transactions + BASE_LABELS = { controller: nil, action: nil }.freeze + THREAD_KEY = :_gitlab_metrics_transaction + METRICS_MUTEX = Mutex.new # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events'.freeze attr_reader :tags, :values, :method, :metrics - attr_accessor :action - def self.current Thread.current[THREAD_KEY] end - # action - A String describing the action performed, usually the class - # plus method name. - def initialize(action = nil) + def initialize @metrics = [] @methods = {} - @started_at = nil + @started_at = nil @finished_at = nil @values = Hash.new(0) - @tags = {} - @action = action + @tags = {} @memory_before = 0 - @memory_after = 0 + @memory_after = 0 end def duration @@ -44,12 +43,15 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = System.monotonic_time + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = System.monotonic_time + @finished_at = System.monotonic_time + + self.class.metric_transaction_duration_seconds.observe(labels, duration * 1000) + self.class.metric_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) Thread.current[THREAD_KEY] = nil end @@ -66,33 +68,29 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - @metrics << Metric.new(EVENT_SERIES, - { count: 1 }, - { event: event_name }.merge(tags), - :event) + self.class.metric_event_counter(event_name, tags).increment(tags.merge(labels)) + @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) end # Returns a MethodCall object for the given name. - def method_call_for(name) + def method_call_for(name, module_name, method_name) unless method = @methods[name] - @methods[name] = method = MethodCall.new(name, Instrumentation.series) + @methods[name] = method = MethodCall.new(name, module_name, method_name, self) end method end - def increment(name, value) + def increment(name, value, use_prometheus = true) + self.class.metric_transaction_counter(name).increment(labels, value) if use_prometheus @values[name] += value end - def set(name, value) + def set(name, value, use_prometheus = true) + self.class.metric_transaction_gauge(name).set(labels, value) if use_prometheus @values[name] = value end - def add_tag(key, value) - @tags[key] = value - end - def finish track_self submit @@ -117,14 +115,83 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - - hash[:tags][:action] ||= @action if @action && !metric.event? + hash[:tags][:action] ||= action if action && !metric.event? hash end Metrics.submit_metrics(submit_hashes) end + + def labels + BASE_LABELS + end + + # returns string describing the action performed, usually the class plus method name. + def action + "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty? + end + + def self.metric_transaction_duration_seconds + return @metric_transaction_duration_seconds if @metric_transaction_duration_seconds + + METRICS_MUTEX.synchronize do + @metric_transaction_duration_seconds ||= Gitlab::Metrics.histogram( + :gitlab_transaction_duration_seconds, + 'Transaction duration', + BASE_LABELS, + [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] + ) + end + end + + def self.metric_transaction_allocated_memory_bytes + return @metric_transaction_allocated_memory_bytes if @metric_transaction_allocated_memory_bytes + + METRICS_MUTEX.synchronize do + @metric_transaction_allocated_memory_bytes ||= Gitlab::Metrics.histogram( + :gitlab_transaction_allocated_memory_bytes, + 'Transaction allocated memory bytes', + BASE_LABELS, + [1000, 10000, 20000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 100000000] + ) + end + end + + def self.metric_event_counter(event_name, tags) + return @metric_event_counters[event_name] if @metric_event_counters&.has_key?(event_name) + + METRICS_MUTEX.synchronize do + @metric_event_counters ||= {} + @metric_event_counters[event_name] ||= Gitlab::Metrics.counter( + "gitlab_transaction_event_#{event_name}_total".to_sym, + "Transaction event #{event_name} counter", + tags.merge(BASE_LABELS) + ) + end + end + + def self.metric_transaction_counter(name) + return @metric_transaction_counters[name] if @metric_transaction_counters&.has_key?(name) + + METRICS_MUTEX.synchronize do + @metric_transaction_counters ||= {} + @metric_transaction_counters[name] ||= Gitlab::Metrics.counter( + "gitlab_transaction_#{name}_total".to_sym, "Transaction #{name} counter", BASE_LABELS + ) + end + end + + def self.metric_transaction_gauge(name) + return @metric_transaction_gauges[name] if @metric_transaction_gauges&.has_key?(name) + + METRICS_MUTEX.synchronize do + @metric_transaction_gauges ||= {} + @metric_transaction_gauges[name] ||= Gitlab::Metrics.gauge( + "gitlab_transaction_#{name}".to_sym, "Transaction gauge #{name}", BASE_LABELS, :livesum + ) + end + end end end end diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb deleted file mode 100644 index f6987252039..00000000000 --- a/lib/gitlab/metrics/unicorn_sampler.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Gitlab - module Metrics - class UnicornSampler < BaseSampler - def initialize(interval) - super(interval) - end - - def unicorn_active_connections - @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) - end - - def unicorn_queued_connections - @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) - end - - def enabled? - # Raindrops::Linux.tcp_listener_stats is only present on Linux - unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) - end - - def sample - Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) - end - - Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| - unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) - unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) - end - end - - private - - def tcp_listeners - @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) - end - - def unix_listeners - @unix_listeners ||= Unicorn.listener_names - tcp_listeners - end - - def unicorn_with_listeners? - defined?(Unicorn) && Unicorn.listener_names.any? - end - end - end -end diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb new file mode 100644 index 00000000000..89ff02a96d6 --- /dev/null +++ b/lib/gitlab/metrics/web_transaction.rb @@ -0,0 +1,82 @@ +module Gitlab + module Metrics + class WebTransaction < Transaction + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + + CONTENT_TYPES = { + 'text/html' => :html, + 'text/plain' => :txt, + 'application/json' => :json, + 'text/js' => :js, + 'application/atom+xml' => :atom, + 'image/png' => :png, + 'image/jpeg' => :jpeg, + 'image/gif' => :gif, + 'image/svg+xml' => :svg + }.freeze + + def initialize(env) + super() + @env = env + end + + def labels + return @labels if @labels + + # memoize transaction labels only source env variables were present + @labels = if @env[CONTROLLER_KEY] + labels_from_controller || {} + elsif @env[ENDPOINT_KEY] + labels_from_endpoint || {} + end + + @labels || {} + end + + private + + def labels_from_controller + controller = @env[CONTROLLER_KEY] + + action = "#{controller.action_name}" + suffix = CONTENT_TYPES[controller.content_type] + + if suffix && suffix != :html + action += ".#{suffix}" + end + + { controller: controller.class.name, action: action } + end + + def labels_from_endpoint + endpoint = @env[ENDPOINT_KEY] + + begin + route = endpoint.route + rescue + # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + end + + if route + path = endpoint_paths_cache[route.request_method][route.path] + { controller: 'Grape', action: "#{route.request_method} #{path}" } + end + end + + def endpoint_paths_cache + @endpoint_paths_cache ||= Hash.new do |hash, http_method| + hash[http_method] = Hash.new do |inner_hash, raw_path| + inner_hash[raw_path] = endpoint_instrumentable_path(raw_path) + end + end + end + + def endpoint_instrumentable_path(raw_path) + raw_path.sub('(.:format)', '').sub('/:version', '') + end + end + end +end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 63c3372da51..bc70b2459ef 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -14,11 +14,22 @@ module Gitlab proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request - trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) + duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 + trans.set(:rails_queue_duration, duration) + metric_rails_queue_duration_seconds.observe(trans.labels, duration / 1_000) end @app.call(env) end + + private + + def metric_rails_queue_duration_seconds + @metric_rails_queue_duration_seconds ||= Gitlab::Metrics.histogram( + :gitlab_rails_queue_duration_seconds, + Gitlab::Metrics::Transaction::BASE_LABELS + ) + end end end end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 8853dfa3d2d..5e4932e4e57 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -66,11 +66,7 @@ module Gitlab end def whitelisted_routes - logout_route || grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def logout_route - route_hash[:controller] == 'sessions' && route_hash[:action] == 'destroy' + grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end def sidekiq_route diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 47c2a422387..b4b3b00c84d 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -179,7 +179,7 @@ module Gitlab valid_username = ::Namespace.clean_path(username) uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !DynamicPathValidator.valid_user_path?(s) } + valid_username = uniquify.string(valid_username) { |s| !UserPathValidator.valid_path?(s) } name = auth_hash.name name = valid_username if name.strip.empty? diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index cd8b2eba6c4..9a3817ff00a 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -113,22 +113,6 @@ module Gitlab # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ - - activity - analytics - audit_events - avatar - edit - group_members - hooks - issues - labels - ldap - ldap_group_links - merge_requests - milestones - notification_setting - pipeline_quota - projects ].freeze ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index bd677ec4bf3..2c7b8af83f2 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -25,7 +25,7 @@ module Gitlab # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex - @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + @container_repository_regex ||= %r{\A[a-z0-9]+((?:[._/]|__|[-])[a-z0-9]+)*\Z} end ## diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index e57890f1143..910533076b0 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -40,5 +40,24 @@ module Gitlab def self.url_helpers @url_helpers ||= Gitlab::Application.routes.url_helpers end + + def self.redirect_legacy_paths(router, *paths) + build_redirect_path = lambda do |request, _params, path| + # Only replace the last occurence of `path`. + # + # `request.fullpath` includes the querystring + path = request.path.sub(%r{/#{path}/*(?!.*#{path})}, "/-/#{path}/") + path << "?#{request.query_string}" if request.query_string.present? + + path + end + + paths.each do |path| + router.match "/#{path}(/*rest)", + via: [:get, :post, :patch, :delete], + to: router.redirect { |params, request| build_redirect_path.call(request, params, path) }, + as: "legacy_#{path}_redirect" + end + end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index fee1a127fd7..13150ddab67 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -22,10 +22,12 @@ module Gitlab return true if blocked_user_or_hostname?(uri.user) return true if blocked_user_or_hostname?(uri.hostname) - server_ips = Resolv.getaddresses(uri.hostname) + server_ips = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM).map(&:ip_address) return true if (blocked_ips & server_ips).any? rescue Addressable::URI::InvalidURIError return true + rescue SocketError + return false end false diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 70a403652e7..112d4939582 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -48,9 +48,9 @@ module Gitlab deploy_keys: DeployKey.count, deployments: Deployment.count, environments: ::Environment.count, - gcp_clusters: ::Gcp::Cluster.count, - gcp_clusters_enabled: ::Gcp::Cluster.enabled.count, - gcp_clusters_disabled: ::Gcp::Cluster.disabled.count, + clusters: ::Clusters::Cluster.count, + clusters_enabled: ::Clusters::Cluster.enabled.count, + clusters_disabled: ::Clusters::Cluster.disabled.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb new file mode 100644 index 00000000000..a2ac9285b56 --- /dev/null +++ b/lib/gitlab/utils/strong_memoize.rb @@ -0,0 +1,31 @@ +module Gitlab + module Utils + module StrongMemoize + # Instead of writing patterns like this: + # + # def trigger_from_token + # return @trigger if defined?(@trigger) + # + # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) + # end + # + # We could write it like: + # + # def trigger_from_token + # strong_memoize(:trigger) do + # Ci::Trigger.find_by_token(params[:token].to_s) + # end + # end + # + def strong_memoize(name) + ivar_name = "@#{name}" + + if instance_variable_defined?(ivar_name) + instance_variable_get(ivar_name) + else + instance_variable_set(ivar_name, yield) + end + end + end + end +end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index a440a3e3562..9242cbe840c 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -3,7 +3,6 @@ require 'google/apis/container_v1' module GoogleApi module CloudPlatform class Client < GoogleApi::Auth - DEFAULT_MACHINE_TYPE = 'n1-standard-1'.freeze SCOPE = 'https://www.googleapis.com/auth/cloud-platform'.freeze LEAST_TOKEN_LIFE_TIME = 10.minutes diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 87ca39b079b..c2d3a6b6950 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,5 +1,28 @@ namespace :gemojione do desc 'Generates Emoji SHA256 digests' + + task aliases: ['yarn:check', 'environment'] do + require 'json' + + aliases = {} + + index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json') + index = JSON.parse(File.read(index_file)) + + index.each_pair do |key, data| + data['aliases'].each do |a| + a.tr!(':', '') + + aliases[a] = key + end + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(aliases, indent: ' ', space: '', space_before: '')) + end + end + task digests: ['yarn:check', 'environment'] do require 'digest/sha2' require 'json' @@ -16,8 +39,13 @@ namespace :gemojione do fpath = File.join(dir, "#{emoji_hash['unicode']}.png") hash_digest = Digest::SHA256.file(fpath).hexdigest + category = emoji_hash['category'] + if name == 'gay_pride_flag' + category = 'flags' + end + entry = { - category: emoji_hash['category'], + category: category, moji: emoji_hash['moji'], description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), @@ -29,7 +57,6 @@ namespace :gemojione do end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - File.open(out, 'w') do |handle| handle.write(JSON.pretty_generate(resultant_emoji_map)) end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 1650263b98d..9dcf44fdc3e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,24 +33,29 @@ namespace :gitlab do backup.unpack unless backup.skipped?('db') - unless ENV['force'] == 'yes' - warning = <<-MSG.strip_heredoc - Before restoring the database we recommend removing all existing - tables to avoid future upgrade problems. Be aware that if you have - custom tables in the GitLab database these tables and all data will be - removed. - MSG - puts warning.color(:red) - ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) + begin + unless ENV['force'] == 'yes' + warning = <<-MSG.strip_heredoc + Before restoring the database, we will remove all existing + tables to avoid future upgrade problems. Be aware that if you have + custom tables in the GitLab database these tables and all data will be + removed. + MSG + puts warning.color(:red) + ask_to_continue + puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + $progress.puts 'Cleaning the database ... '.color(:blue) + Rake::Task['gitlab:db:drop_tables'].invoke + $progress.puts 'done'.color(:green) + Rake::Task['gitlab:backup:db:restore'].invoke + rescue Gitlab::TaskAbortedByUserError + puts "Quitting...".color(:red) + exit 1 end - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - $progress.puts 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - $progress.puts 'done'.color(:green) - Rake::Task['gitlab:backup:db:restore'].invoke end Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index d227a0c8bdb..adfcc3cda22 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,23 +2,21 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) - # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production + # 1. copy the bare repos to a specific path that contain the group or subgroups structure as folders + # 2. run: bundle exec rake gitlab:import:repos[/path/to/repos] RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped - # - # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" - task repos: :environment do - if Project.current_application_settings.hashed_storage_enabled - puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red) + task :repos, [:import_path] => :environment do |_t, args| + unless args.import_path + puts 'Please specify an import path that contains the repositories'.color(:red) exit 1 end - Gitlab::BareRepositoryImporter.execute + Gitlab::BareRepositoryImport::Importer.execute(args.import_path) end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 7f86fd7b45e..aafbe52e5f8 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,14 +7,16 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { token: token, verbose: true } + @options = { token: token } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first end def run! - @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! + @repo = GithubRepos + .new(@options[:token], @current_user, @github_repo) + .choose_one! raise 'No repo found!' unless @repo @@ -28,7 +30,7 @@ class GithubImport private def show_warning! - puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo.full_name.bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch @@ -42,7 +44,9 @@ class GithubImport import_success = false timings = Benchmark.measure do - import_success = Github::Import.new(@project, @options).execute + import_success = Gitlab::GithubImport::SequentialImporter + .new(@project, token: @options[:token]) + .execute end if import_success @@ -63,16 +67,16 @@ class GithubImport @current_user, name: name, path: name, - description: @repo['description'], + description: @repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - skip_wiki: @repo['has_wiki'] + skip_wiki: @repo.has_wiki ).execute project.update!( import_type: 'github', - import_source: @repo['full_name'], - import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@") + import_source: @repo.full_name, + import_url: @repo.clone_url.sub('://', "://#{@options[:token]}@") ) project @@ -91,13 +95,15 @@ class GithubImport end def visibility_level - @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.current_application_settings.default_project_visibility end end class GithubRepos - def initialize(options, current_user, github_repo) - @options = options + def initialize(token, current_user, github_repo) + @client = Gitlab::GithubImport::Client.new(token) + @client.octokit.auto_paginate = true + @current_user = current_user @github_repo = github_repo end @@ -106,17 +112,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo['id'].to_s.bright}".color(:green) - print "\tName: #{repo['full_name']}\n".color(:green) + print "ID: #{repo.id.to_s.bright}".color(:green) + print "\tName: #{repo.full_name}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo['id'] == repo_id } + repos.find { |repo| repo.id == repo_id } end def found_github_repo - repos.find { |repo| repo['full_name'] == @github_repo } + repos.find { |repo| repo.full_name == @github_repo } end def repo_id @@ -124,7 +130,7 @@ class GithubRepos end def repos - Github::Repositories.new(@options).fetch + @client.octokit.list_repositories end end |