diff options
| author | Filipa Lacerda <filipa@gitlab.com> | 2017-12-11 18:56:53 +0000 |
|---|---|---|
| committer | Filipa Lacerda <filipa@gitlab.com> | 2017-12-11 18:56:53 +0000 |
| commit | fcd44484cd427fc75f05e1c8363e9def4b4bf64f (patch) | |
| tree | e3181efad22ad409ef98a89592c1911d43f7bdad /lib | |
| parent | 7c68be16a6d364cd40d9fbb6e0878b9f2947bb88 (diff) | |
| parent | 4ccbd556d98e002b1c521fd3dd7748fe1d9c4044 (diff) | |
| download | gitlab-ce-28869-es6-modules.tar.gz | |
Merge branch 'master' into 28869-es6-modules28869-es6-modules
* master: (117 commits)
small change to make less conflict with EE version
Add cop for use of remove_column
Resolve merge conflicts with dev.gitlab.org/master after security release
add index for doc/administration/operations/
Remove RubySampler#sample_objects for performance as well
Bugfix: User can't change the access level of an access requester
Add spec for removing issues.assignee_id
updated imports
Keep track of storage check timings
Remove a header level in the new 'Automatic CE->EE merge' doc
Improve down step of removing issues.assignee_id column
Fix specs after removing assignee_id field
Remove issues.assignee_id column
Resolve conflicts in app/models/user.rb
Fix image view mode
Do not raise when downstream pipeline is created
Remove the need for destroy and add a comment in the spec
Use build instead of create in importer spec
Simplify normalizing of paths
Remove allocation tracking code from InfluxDB sampler for performance
...
Diffstat (limited to 'lib')
58 files changed, 912 insertions, 462 deletions
diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index 118883f5ea5..598c76f6168 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -41,7 +41,7 @@ module API detail 'This feature was introduced in GitLab 9.5' end delete do - Gitlab::Git::Storage::CircuitBreaker.reset_all! + Gitlab::Git::Storage::FailureInfo.reset_all! end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 62ee20bf7de..928706dfda7 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -16,10 +16,13 @@ module API class UserBasic < UserSafe expose :state + expose :avatar_url do |user, options| user.avatar_url(only_path: false) end + expose :avatar_path, if: ->(user, options) { options.fetch(:only_path, false) && user.avatar_path } + expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) end @@ -245,8 +248,21 @@ module API end class GroupDetail < Group - expose :projects, using: Entities::Project - expose :shared_projects, using: Entities::Project + expose :projects, using: Entities::Project do |group, options| + GroupProjectsFinder.new( + group: group, + current_user: options[:current_user], + options: { only_owned: true } + ).execute + end + + expose :shared_projects, using: Entities::Project do |group, options| + GroupProjectsFinder.new( + group: group, + current_user: options[:current_user], + options: { only_shared: true } + ).execute + end end class Commit < Grape::Entity diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 451121a4cea..ccaaeca10d4 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -4,6 +4,7 @@ module API before { authenticate_by_gitlab_shell_token! } helpers ::API::Helpers::InternalHelpers + helpers ::Gitlab::Identifier namespace 'internal' do # Check if git command is allowed to project @@ -176,17 +177,25 @@ module API post '/post_receive' do status 200 - PostReceive.perform_async(params[:gl_repository], params[:identifier], params[:changes]) broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease - { + output = { merge_request_urls: merge_request_urls, broadcast_message: broadcast_message, reference_counter_decreased: reference_counter_decreased } + + project = Gitlab::GlRepository.parse(params[:gl_repository]).first + user = identify(params[:identifier]) + redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + if redirect_message + output[:redirected_message] = redirect_message + end + + output end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e60e00d7956..5f943ba27d1 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -161,6 +161,8 @@ module API use :issue_params end post ':id/issues' do + authorize! :create_issue, user_project + # Setting created_at time only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index b5021e8a712..614822509f0 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -39,10 +39,10 @@ module API end params do requires :name, type: String, desc: 'The name of the protected branch' - optional :push_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :push_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to push (defaults: `40`, master access level)' - optional :merge_access_level, type: Integer, default: Gitlab::Access::MASTER, + optional :merge_access_level, type: Integer, values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, desc: 'Access levels allowed to merge (defaults: `40`, master access level)' end @@ -52,15 +52,13 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - protected_branch_params = { - name: params[:name], - push_access_levels_attributes: [{ access_level: params[:push_access_level] }], - merge_access_levels_attributes: [{ access_level: params[:merge_access_level] }] - } + # Replace with `declared(params)` after updating to grape v1.0.2 + # See https://github.com/ruby-grape/grape/pull/1710 + # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 + declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - service_args = [user_project, current_user, protected_branch_params] - - protected_branch = ::ProtectedBranches::CreateService.new(*service_args).execute + api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) + protected_branch = api_service.create if protected_branch.persisted? present protected_branch, with: Entities::ProtectedBranch, project: user_project diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index b6d273b98c2..2a04c03919d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -193,12 +193,9 @@ module Backup end def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_exists_cache # protect backups from stale cache - project_or_wiki.repository.empty_repo? - rescue => e - progress.puts "Ignoring repository error and continuing backing up project: #{display_repo_path(project_or_wiki)} - #{e.message}".color(:orange) - - false + # Protect against stale caches + project_or_wiki.repository.expire_emptiness_caches + project_or_wiki.repository.empty? end def repository_storage_paths_args diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index e2b57adf611..d8fb7705b2a 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -11,7 +11,7 @@ module Banzai # ref - String reference. # # Returns a Project, or nil if the reference can't be found - def project_from_ref(ref) + def parent_from_ref(ref) return context[:project] unless ref Project.find_by_full_path(ref) diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 8975395aff1..e7e6a90b5fd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -82,9 +82,9 @@ module Banzai end end - def project_from_ref_cached(ref) - cached_call(:banzai_project_refs, ref) do - project_from_ref(ref) + def from_ref_cached(ref) + cached_call("banzai_#{parent_type}_refs".to_sym, ref) do + parent_from_ref(ref) end end @@ -153,15 +153,20 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| - project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref_cached(project_path) + parent_path = if parent_type == :group + full_group_path(namespace_ref) + else + full_project_path(namespace_ref, project_ref) + end - if project + parent = from_ref_cached(parent_path) + + if parent object = if link_reference - find_object_from_link_cached(project, id) + find_object_from_link_cached(parent, id) else - find_object_cached(project, id) + find_object_cached(parent, id) end end @@ -169,13 +174,13 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object, link: !!link_content) + data = data_attributes_for(link_content || match, parent, object, link: !!link_content) url = if matches.names.include?("url") && matches[:url] matches[:url] else - url_for_object_cached(object, project) + url_for_object_cached(object, parent) end content = link_content || object_link_text(object, matches) @@ -224,17 +229,24 @@ module Banzai # Returns a Hash containing all object references (e.g. issue IDs) per the # project they belong to. - def references_per_project - @references_per_project ||= begin + def references_per_parent + @references_per ||= {} + + @references_per[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } regex = Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) nodes.each do |node| node.to_html.scan(regex) do - project_path = full_project_path($~[:namespace], $~[:project]) + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + symbol = $~[object_sym] - refs[project_path] << symbol if object_class.reference_valid?(symbol) + refs[path] << symbol if object_class.reference_valid?(symbol) end end @@ -244,35 +256,41 @@ module Banzai # Returns a Hash containing referenced projects grouped per their full # path. - def projects_per_reference - @projects_per_reference ||= begin + def parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin refs = Set.new - references_per_project.each do |project_ref, _| - refs << project_ref + references_per_parent.each do |ref, _| + refs << ref end - find_projects_for_paths(refs.to_a).index_by(&:full_path) + find_for_paths(refs.to_a).index_by(&:full_path) end end - def projects_relation_for_paths(paths) - Project.where_full_path_in(paths).includes(:namespace) + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(:namespace) if parent_type == :project end # Returns projects for the given paths. - def find_projects_for_paths(paths) + def find_for_paths(paths) if RequestStore.active? - cache = project_refs_cache + cache = refs_cache to_query = paths - cache.keys unless to_query.empty? - projects = projects_relation_for_paths(to_query) + records = relation_for_paths(to_query) found = [] - projects.each do |project| - ref = project.full_path - get_or_set_cache(cache, ref) { project } + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } found << ref end @@ -284,33 +302,37 @@ module Banzai cache.slice(*paths).values.compact else - projects_relation_for_paths(paths) + relation_for_paths(paths) end end - def current_project_path - return unless project - - @current_project_path ||= project.full_path + def current_parent_path + @current_parent_path ||= parent&.full_path end def current_project_namespace_path - return unless project - - @current_project_namespace_path ||= project.namespace.full_path + @current_project_namespace_path ||= project&.namespace&.full_path end private def full_project_path(namespace, project_ref) - return current_project_path unless project_ref + return current_parent_path unless project_ref namespace_ref = namespace || current_project_namespace_path "#{namespace_ref}/#{project_ref}" end - def project_refs_cache - RequestStore[:banzai_project_refs] ||= {} + def refs_cache + RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + + def parent_type + :project + end + + def parent + parent_type == :project ? project : group end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 714e0319025..eedb95197aa 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -22,10 +22,30 @@ module Banzai end end + def referenced_merge_request_commit_shas + return [] unless noteable.is_a?(MergeRequest) + + @referenced_merge_request_commit_shas ||= begin + referenced_shas = references_per_parent.values.reduce(:|).to_a + noteable.all_commit_shas.select do |sha| + referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } + end + end + end + def url_for_object(commit, project) h = Gitlab::Routing.url_helpers - h.project_commit_url(project, commit, - only_path: context[:only_path]) + + if referenced_merge_request_commit_shas.include?(commit.id) + h.diffs_project_merge_request_url(project, + noteable, + commit_id: commit.id, + only_path: only_path?) + else + h.project_commit_url(project, + commit, + only_path: only_path?) + end end def object_link_text_extras(object, matches) @@ -38,6 +58,16 @@ module Banzai extras end + + private + + def noteable + context[:noteable] + end + + def only_path? + context[:only_path] + end end end end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb new file mode 100644 index 00000000000..265924abe24 --- /dev/null +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -0,0 +1,12 @@ +module Banzai + module Filter + # The actual filter is implemented in the EE mixin + class EpicReferenceFilter < IssuableReferenceFilter + self.reference_type = :epic + + def self.object_class + Epic + end + end + end +end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb new file mode 100644 index 00000000000..7addf09be73 --- /dev/null +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + class IssuableReferenceFilter < AbstractReferenceFilter + def records_per_parent + @records_per_project ||= {} + + @records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record.iid.to_i] = record + end + end + + hash + end + end + + def find_object(parent, iid) + records_per_parent[parent][iid] + end + + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index ce1ab977d3b..6877cae8c55 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -8,46 +8,24 @@ module Banzai # When external issues tracker like Jira is activated we should not # use issue reference pattern, but we should still be able # to reference issues from other GitLab projects. - class IssueReferenceFilter < AbstractReferenceFilter + class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue def self.object_class Issue end - def find_object(project, iid) - issues_per_project[project][iid] - end - def url_for_object(issue, project) IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the issues per Project instance. - def issues_per_project - @issues_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - issue_ids = references_per_project[path] - issues = project.issues.where(iid: issue_ids.to_a) - - issues.each do |issue| - hash[project][issue.iid.to_i] = issue - end - end - - hash - end - end - def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end + + def parent_records(parent, ids) + parent.issues.where(iid: ids.to_a) + end end end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 5364984c9d3..d5360ad8f68 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai end def find_label(project_ref, label_id, label_name) - project = project_from_ref(project_ref) + project = parent_from_ref(project_ref) return unless project label_params = label_params(label_id, label_name) @@ -66,7 +66,7 @@ module Banzai def object_link_text(object, matches) project_path = full_project_path(matches[:namespace], matches[:project]) - project_from_ref = project_from_ref_cached(project_path) + project_from_ref = from_ref_cached(project_path) reference = project_from_ref.to_human_reference(project) label_suffix = " <i>in #{reference}</i>" if reference.present? diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 0eab865ac04..b3cfa97d0e0 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -4,48 +4,19 @@ module Banzai # to merge requests that do not exist are ignored. # # This filter supports cross-project references. - class MergeRequestReferenceFilter < AbstractReferenceFilter + class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request def self.object_class MergeRequest end - def find_object(project, iid) - merge_requests_per_project[project][iid] - end - def url_for_object(mr, project) h = Gitlab::Routing.url_helpers h.project_merge_request_url(project, mr, only_path: context[:only_path]) end - def project_from_ref(ref) - projects_per_reference[ref || current_project_path] - end - - # Returns a Hash containing the merge_requests per Project instance. - def merge_requests_per_project - @merge_requests_per_project ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - projects_per_reference.each do |path, project| - merge_request_ids = references_per_project[path] - - merge_requests = project.merge_requests - .where(iid: merge_request_ids.to_a) - .includes(target_project: :namespace) - - merge_requests.each do |merge_request| - hash[project][merge_request.iid.to_i] = merge_request - end - end - - hash - end - end - def object_link_text_extras(object, matches) extras = super @@ -61,6 +32,12 @@ module Banzai extras end + + def parent_records(parent, ids) + parent.merge_requests + .where(iid: ids.to_a) + .includes(target_project: :namespace) + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index bb5da310e09..2a6b0964ac5 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -38,7 +38,7 @@ module Banzai def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) project_path = full_project_path(namespace_ref, project_ref) - project = project_from_ref(project_path) + project = parent_from_ref(project_path) return unless project diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 47151626208..97244159985 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -32,6 +32,7 @@ module Banzai .gsub(PUNCTUATION_REGEXP, '') # remove punctuation .tr(' ', '-') # replace spaces with dash .squeeze('-') # replace multiple dashes with one + .gsub(/\A(\d+)\z/, 'anchor-\1') # digits-only hrefs conflict with issue refs uniq = headers[id] > 0 ? "-#{headers[id]}" : '' headers[id] += 1 diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 09844931be5..d64f9ac4eb6 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -8,7 +8,7 @@ module Banzai # class UploadLinkFilter < HTML::Pipeline::Filter def call - return doc unless project + return doc unless project || group doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| process_link_attr el.attribute('href') @@ -28,13 +28,27 @@ module Banzai end def build_url(uri) - File.join(Gitlab.config.gitlab.url, project.full_path, uri) + base_path = Gitlab.config.gitlab.url + + if group + urls = Gitlab::Routing.url_helpers + # we need to get last 2 parts of the uri which are secret and filename + uri_parts = uri.split(File::SEPARATOR) + file_path = urls.show_group_uploads_path(group, uri_parts[-2], uri_parts[-1]) + File.join(base_path, file_path) + else + File.join(base_path, project.full_path, uri) + end end def project context[:project] end + def group + context[:group] + end + # Ensure that a :project key exists in context # # Note that while the key might exist, its value could be nil! diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index cbabf9156de..49603d0b363 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,8 +28,8 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( - merge_request_parser.merge_requests_for_nodes(nodes) + issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( + merge_request_parser.records_for_nodes(nodes) ) # The project for the issue/MR might be pending for deletion! diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index ecb3affbba5..2691be81623 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -17,11 +17,11 @@ module Banzai # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use during redaction + # redaction_context - A Hash containing extra attributes to use during redaction def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @redaction_context = redaction_context + @redaction_context = base_context.merge(redaction_context) end # Renders and redacts an Array of objects. @@ -73,19 +73,19 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - base_context.merge(object.banzai_render_context(attribute)) + @redaction_context.merge(object.banzai_render_context(attribute)) end def base_context - @base_context ||= @redaction_context.merge( + { current_user: user, project: project, skip_redaction: true - ) + } end def save_options - return {} unless base_context[:xhtml] + return {} unless @redaction_context[:xhtml] { save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML } end diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb new file mode 100644 index 00000000000..08b8a4c9a0f --- /dev/null +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -0,0 +1,12 @@ +module Banzai + module ReferenceParser + # The actual parser is implemented in the EE mixin + class EpicParser < IssuableParser + self.reference_type = :epic + + def records_for_nodes(_nodes) + {} + end + end + end +end diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb new file mode 100644 index 00000000000..3953867eb83 --- /dev/null +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -0,0 +1,25 @@ +module Banzai + module ReferenceParser + class IssuableParser < BaseParser + def nodes_visible_to_user(user, nodes) + records = records_for_nodes(nodes) + + nodes.select do |node| + issuable = records[node] + + issuable && can_read_reference?(user, issuable) + end + end + + def referenced_by(nodes) + records = records_for_nodes(nodes) + + nodes.map { |node| records[node] }.compact.uniq + end + + def can_read_reference?(user, issuable) + can?(user, "read_#{issuable.class.to_s.underscore}".to_sym, issuable) + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index e0a8ca653cb..38d4e3f3e44 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -1,10 +1,10 @@ module Banzai module ReferenceParser - class IssueParser < BaseParser + class IssueParser < IssuableParser self.reference_type = :issue def nodes_visible_to_user(user, nodes) - issues = issues_for_nodes(nodes) + issues = records_for_nodes(nodes) readable_issues = Ability .issues_readable_by_user(issues.values, user).to_set @@ -14,13 +14,7 @@ module Banzai end end - def referenced_by(nodes) - issues = issues_for_nodes(nodes) - - nodes.map { |node| issues[node] }.compact.uniq - end - - def issues_for_nodes(nodes) + def records_for_nodes(nodes) @issues_for_nodes ||= grouped_objects_for_nodes( nodes, Issue.all.includes( diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 75cbc7fdac4..a370ff5b5b3 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -1,25 +1,9 @@ module Banzai module ReferenceParser - class MergeRequestParser < BaseParser + class MergeRequestParser < IssuableParser self.reference_type = :merge_request - def nodes_visible_to_user(user, nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.select do |node| - merge_request = merge_requests[node] - - merge_request && can?(user, :read_merge_request, merge_request.project) - end - end - - def referenced_by(nodes) - merge_requests = merge_requests_for_nodes(nodes) - - nodes.map { |node| merge_requests[node] }.compact.uniq - end - - def merge_requests_for_nodes(nodes) + def records_for_nodes(nodes) @merge_requests_for_nodes ||= grouped_objects_for_nodes( nodes, MergeRequest.includes( @@ -40,10 +24,6 @@ module Banzai self.class.data_attribute ) end - - def can_read_reference?(user, ref_project, node) - can?(user, :read_merge_request, ref_project) - end end end end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 196de667805..298409d8b5a 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -55,6 +55,7 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, + import_type: 'gitlab_project', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index 8574ac6eb30..fa7891c8dcc 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -7,6 +7,8 @@ module Gitlab @root_path = root_path @repo_path = repo_path + @root_path << '/' unless root_path.ends_with?('/') + # Split path into 'all/the/namespaces' and 'project_name' @group_path, _, @project_name = repo_relative_path.rpartition('/') end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb new file mode 100644 index 00000000000..3a1c0a3455e --- /dev/null +++ b/lib/gitlab/checks/project_moved.rb @@ -0,0 +1,65 @@ +module Gitlab + module Checks + class ProjectMoved + REDIRECT_NAMESPACE = "redirect_namespace".freeze + + def initialize(project, user, redirected_path, protocol) + @project = project + @user = user + @redirected_path = redirected_path + @protocol = protocol + end + + def self.fetch_redirect_message(user_id, project_id) + redirect_key = redirect_message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(redirect_key) + redis.del(redirect_key) + message + end + end + + def add_redirect_message + Gitlab::Redis::SharedState.with do |redis| + key = self.class.redirect_message_key(user.id, project.id) + redis.setex(key, 5.minutes, redirect_message) + end + end + + def redirect_message(rejected: false) + <<~MESSAGE.strip_heredoc + Project '#{redirected_path}' was moved to '#{project.full_path}'. + + Please update your Git remote: + + #{remote_url_message(rejected)} + MESSAGE + end + + def permanent_redirect? + RedirectRoute.permanent.exists?(path: redirected_path) + end + + private + + attr_reader :project, :redirected_path, :protocol, :user + + def self.redirect_message_key(user_id, project_id) + "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" + end + + def remote_url_message(rejected) + if rejected + "git remote set-url origin #{url} and try again." + else + "git remote set-url origin #{url}" + end + end + + def url + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index 8d82e1b288d..efed19da21c 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -3,14 +3,13 @@ module Gitlab module Pipeline module Chain class Base - attr_reader :pipeline, :project, :current_user + attr_reader :pipeline, :command + + delegate :project, :current_user, to: :command def initialize(pipeline, command) @pipeline = pipeline @command = command - - @project = command.project - @current_user = command.current_user end def perform! diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index a126dded1ae..70732d26bbd 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -3,20 +3,18 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base - include Chain::Helpers - def perform! @pipeline.assign_attributes( source: @command.source, - project: @project, - ref: ref, - sha: sha, - before_sha: before_sha, - tag: tag_exists?, + project: @command.project, + ref: @command.ref, + sha: @command.sha, + before_sha: @command.before_sha, + tag: @command.tag_exists?, trigger_requests: Array(@command.trigger_request), - user: @current_user, + user: @command.current_user, pipeline_schedule: @command.schedule, - protected: protected_ref? + protected: @command.protected_ref? ) @pipeline.set_config_source @@ -25,32 +23,6 @@ module Gitlab def break? false end - - private - - def ref - @ref ||= Gitlab::Git.ref_name(origin_ref) - end - - def sha - @project.commit(origin_sha || origin_ref).try(:id) - end - - def origin_ref - @command.origin_ref - end - - def origin_sha - @command.checkout_sha || @command.after_sha - end - - def before_sha - @command.checkout_sha || @command.before_sha || Gitlab::Git::BLANK_SHA - end - - def protected_ref? - @project.protected_for?(ref) - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb new file mode 100644 index 00000000000..7b19b10e05b --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -0,0 +1,61 @@ +module Gitlab + module Ci + module Pipeline + module Chain + Command = Struct.new( + :source, :project, :current_user, + :origin_ref, :checkout_sha, :after_sha, :before_sha, + :trigger_request, :schedule, + :ignore_skip_ci, :save_incompleted, + :seeds_block + ) do + include Gitlab::Utils::StrongMemoize + + def initialize(**params) + params.each do |key, value| + self[key] = value + end + end + + def branch_exists? + strong_memoize(:is_branch) do + project.repository.branch_exists?(ref) + end + end + + def tag_exists? + strong_memoize(:is_tag) do + project.repository.tag_exists?(ref) + end + end + + def ref + strong_memoize(:ref) do + Gitlab::Git.ref_name(origin_ref) + end + end + + def sha + strong_memoize(:sha) do + project.commit(origin_sha || origin_ref).try(:id) + end + end + + def origin_sha + checkout_sha || after_sha + end + + def before_sha + self[:before_sha] || checkout_sha || Gitlab::Git::BLANK_SHA + end + + def protected_ref? + strong_memoize(:protected_ref) do + project.protected_for?(ref) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 02d81286f21..bf1380a1da9 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -3,18 +3,6 @@ module Gitlab module Pipeline module Chain module Helpers - def branch_exists? - return @is_branch if defined?(@is_branch) - - @is_branch = project.repository.branch_exists?(pipeline.ref) - end - - def tag_exists? - return @is_tag if defined?(@is_tag) - - @is_tag = project.repository.tag_exists?(pipeline.ref) - end - def error(message) pipeline.errors.add(:base, message) end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 4913a604079..13c6fedd831 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -14,7 +14,7 @@ module Gitlab unless allowed_to_trigger_pipeline? if can?(current_user, :create_pipeline, project) - return error("Insufficient permissions for protected ref '#{pipeline.ref}'") + return error("Insufficient permissions for protected ref '#{command.ref}'") else return error('Insufficient permissions to create a new pipeline') end @@ -29,7 +29,7 @@ module Gitlab if current_user allowed_to_create? else # legacy triggers don't have a corresponding user - !project.protected_for?(@pipeline.ref) + !@command.protected_ref? end end @@ -38,10 +38,10 @@ module Gitlab access = Gitlab::UserAccess.new(current_user, project: project) - if branch_exists? - access.can_update_branch?(@pipeline.ref) - elsif tag_exists? - access.can_create_tag?(@pipeline.ref) + if @command.branch_exists? + access.can_update_branch?(@command.ref) + elsif @command.tag_exists? + access.can_create_tag?(@command.ref) else true # Allow it for now and we'll reject when we check ref existence end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 70a4cfdbdea..9699c24e5b6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -7,14 +7,11 @@ module Gitlab include Chain::Helpers def perform! - unless branch_exists? || tag_exists? + unless @command.branch_exists? || @command.tag_exists? return error('Reference not found') end - ## TODO, we check commit in the service, that is why - # there is no repository access here. - # - unless pipeline.sha + unless @command.sha return error('Commit not found') end end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index c98eefbce25..88e0db830f6 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -13,9 +13,9 @@ module Gitlab def ==(other) other.is_a?(self.class) && - shas_equal?(base_sha, other.base_sha) && - shas_equal?(start_sha, other.start_sha) && - shas_equal?(head_sha, other.head_sha) + Git.shas_eql?(base_sha, other.base_sha) && + Git.shas_eql?(start_sha, other.start_sha) && + Git.shas_eql?(head_sha, other.head_sha) end alias_method :eql?, :== @@ -47,22 +47,6 @@ module Gitlab CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) end end - - private - - def shas_equal?(sha1, sha2) - return true if sha1 == sha2 - return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class - - length = [sha1.length, sha2.length].min - - # If either of the shas is below the minimum length, we cannot be sure - # that they actually refer to the same commit because of hash collision. - return false if length < Commit::MIN_SHA_LENGTH - - sha1[0, length] == sha2[0, length] - end end end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 2d7b57120a6..54783a07919 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -70,7 +70,7 @@ module Gitlab def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object("") { |line, s| s << line[0] }.gsub(/[^ +-]/, ' ') + line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') changed_line_pairs = [] line_prefixes.scan(LINE_PAIRS_PATTERN) do diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 4a9d3e52fae..37face8e7d0 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -280,7 +280,7 @@ module Gitlab The `#{branch}` branch applies cleanly to EE/master! Much ❤️! For more information, see - https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html #{THANKS_FOR_READING_BANNER} } end @@ -357,7 +357,7 @@ module Gitlab Once this is done, you can retry this failed build, and it should pass. Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html #{THANKS_FOR_READING_BANNER} } end @@ -378,7 +378,7 @@ module Gitlab retry this build. Stay 💪 ! For more information, see - https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html #{THANKS_FOR_READING_BANNER} } end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f31cdbc96d..1f7c35cafaa 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -70,6 +70,18 @@ module Gitlab def diff_line_code(file_path, new_line_position, old_line_position) "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" end + + def shas_eql?(sha1, sha2) + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + length = [sha1.length, sha2.length].min + return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH + + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 8900e2d7afe..e90b158fb34 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index e36d5410431..7e8fe173056 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -83,7 +83,7 @@ module Gitlab Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - start_branch_name = nil if start_repository.empty_repo? + start_branch_name = nil if start_repository.empty? if start_branch_name && !start_repository.branch_exists?(start_branch_name) raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index 3685aa20669..6bd6e58feeb 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -24,10 +24,12 @@ module Gitlab @path = repository.path end - def empty_repo? + def empty? # We will override this implementation in gitaly-ruby because we cannot # use '@repository' there. - @repository.empty_repo? + # + # Caches and memoization used on the Rails side + !@repository.exists? || @repository.empty? end def commit_id(revision) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1468069a991..73889328f36 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -75,9 +75,6 @@ module Gitlab @attributes = Gitlab::Git::Attributes.new(path) end - delegate :empty?, - to: :rugged - def ==(other) path == other.path end @@ -206,6 +203,13 @@ module Gitlab end end + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + alias_method :has_visible_content?, :has_local_branches? + def has_local_branches_rugged? rugged.branches.each(:local).any? do |ref| begin @@ -1004,7 +1008,7 @@ module Gitlab Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - return yield nil if start_repository.empty_repo? + return yield nil if start_repository.empty? if start_repository.same_repository?(self) yield commit(start_branch_name) @@ -1120,24 +1124,8 @@ module Gitlab Gitlab::Git::Commit.find(self, ref) end - # Refactoring aid; allows us to copy code from app/models/repository.rb - def empty_repo? - !exists? || !has_visible_content? - end - - # - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - # - # This method return true if repository contains some content visible in project page. - # - def has_visible_content? - return @has_visible_content if defined?(@has_visible_content) - - @has_visible_content = has_local_branches? + def empty? + !has_visible_content? end # Like all public `Gitlab::Git::Repository` methods, this method is part @@ -1172,9 +1160,15 @@ module Gitlab end def fsck - output, status = run_git(%W[--git-dir=#{path} fsck], nice: true) + gitaly_migrate(:git_fsck) do |is_enabled| + msg, status = if is_enabled + gitaly_fsck + else + shell_fsck + end - raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero? + raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? + end end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) @@ -1322,6 +1316,14 @@ module Gitlab File.write(File.join(worktree_info_path, 'sparse-checkout'), files) end + def gitaly_fsck + gitaly_repository_client.fsck + end + + def shell_fsck + run_git(%W[--git-dir=#{path} fsck], nice: true) + end + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb new file mode 100644 index 00000000000..d3c37f82101 --- /dev/null +++ b/lib/gitlab/git/storage/checker.rb @@ -0,0 +1,120 @@ +module Gitlab + module Git + module Storage + class Checker + include CircuitBreakerSettings + + attr_reader :storage_path, :storage, :hostname, :logger + METRICS_MUTEX = Mutex.new + STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze + + def self.check_all(logger = Rails.logger) + threads = Gitlab.config.repositories.storages.keys.map do |storage_name| + Thread.new do + Thread.current[:result] = new(storage_name, logger).check_with_lease + end + end + + threads.map do |thread| + thread.join + thread[:result] + end + end + + def self.check_histogram + @check_histogram ||= + METRICS_MUTEX.synchronize do + @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds, + 'Storage check time in seconds', + {}, + STORAGE_TIMING_BUCKETS + ) + end + end + + def initialize(storage, logger = Rails.logger) + @storage = storage + config = Gitlab.config.repositories.storages[@storage] + @storage_path = config['path'] + @logger = logger + + @hostname = Gitlab::Environment.hostname + end + + def check_with_lease + lease_key = "storage_check:#{cache_key}" + lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) + result = { storage: storage, success: nil } + + if uuid = lease.try_obtain + result[:success] = check + + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + else + logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") + end + + result + end + + def check + if perform_access_check + track_storage_accessible + true + else + track_storage_inaccessible + logger.error("#{hostname}: #{storage}: Not accessible.") + false + end + end + + private + + def perform_access_check + start_time = Gitlab::Metrics::System.monotonic_time + + Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) + ensure + execution_time = Gitlab::Metrics::System.monotonic_time - start_time + self.class.check_histogram.observe({ storage: storage }, execution_time) + end + + def track_storage_inaccessible + first_failure = current_failure_info.first_failure || Time.now + last_failure = Time.now + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, first_failure.to_i) + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hincrby(cache_key, :failure_count, 1) + redis.expire(cache_key, failure_reset_time) + maintain_known_keys(redis) + end + end + end + + def track_storage_accessible + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :first_failure, nil) + redis.hset(cache_key, :last_failure, nil) + redis.hset(cache_key, :failure_count, 0) + maintain_known_keys(redis) + end + end + end + + def maintain_known_keys(redis) + expire_time = Time.now.to_i + failure_reset_time + redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) + redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) + end + + def current_failure_info + FailureInfo.load(cache_key) + end + end + end + end +end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 4328c0ea29b..898bb1b65be 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -4,22 +4,11 @@ module Gitlab class CircuitBreaker include CircuitBreakerSettings - FailureInfo = Struct.new(:last_failure, :failure_count) - attr_reader :storage, - :hostname, - :storage_path - - delegate :last_failure, :failure_count, to: :failure_info - - def self.reset_all! - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end + :hostname - RequestStore.delete(:circuitbreaker_cache) - end + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do @@ -46,9 +35,6 @@ module Gitlab def initialize(storage, hostname) @storage = storage @hostname = hostname - - config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] end def perform @@ -65,15 +51,6 @@ module Gitlab failure_count > failure_count_threshold end - def backing_off? - return false if no_failures? - - recent_failure = last_failure > failure_wait_time.seconds.ago - too_many_failures = failure_count > backoff_threshold - - recent_failure && too_many_failures - end - private # The circuitbreaker can be enabled for the entire fleet using a Feature @@ -86,88 +63,13 @@ module Gitlab end def failure_info - @failure_info ||= get_failure_info - end - - # Memoizing the `storage_available` call means we only do it once per - # request when the storage is available. - # - # When the storage appears not available, and the memoized value is `false` - # we might want to try again. - def storage_available? - return @storage_available if @storage_available - - if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck - .storage_available?(storage_path, storage_timeout, access_retries) - track_storage_accessible - else - track_storage_inaccessible - end - - @storage_available + @failure_info ||= FailureInfo.load(cache_key) end def check_storage_accessible! if circuit_broken? raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) end - - if backing_off? - raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time) - end - - unless storage_available? - raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time) - end - end - - def no_failures? - last_failure.blank? && failure_count == 0 - end - - def track_storage_inaccessible - @failure_info = FailureInfo.new(Time.now, failure_count + 1) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - maintain_known_keys(redis) - end - end - end - - def track_storage_accessible - @failure_info = FailureInfo.new(nil, 0) - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - maintain_known_keys(redis) - end - end - end - - def maintain_known_keys(redis) - expire_time = Time.now.to_i + failure_reset_time - redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) - redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) - end - - def get_failure_info - last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - - FailureInfo.new(last_failure, failure_count.to_i) - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end end end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb index 257fe8cd8f0..c9e225f187d 100644 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -6,10 +6,6 @@ module Gitlab application_settings.circuitbreaker_failure_count_threshold end - def failure_wait_time - application_settings.circuitbreaker_failure_wait_time - end - def failure_reset_time application_settings.circuitbreaker_failure_reset_time end @@ -22,8 +18,12 @@ module Gitlab application_settings.circuitbreaker_access_retries end - def backoff_threshold - application_settings.circuitbreaker_backoff_threshold + def check_interval + application_settings.circuitbreaker_check_interval + end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end private diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb new file mode 100644 index 00000000000..387279c110d --- /dev/null +++ b/lib/gitlab/git/storage/failure_info.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + module Storage + class FailureInfo + attr_accessor :first_failure, :last_failure, :failure_count + + def self.reset_all! + Gitlab::Git::Storage.redis.with do |redis| + all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) + redis.del(*all_storage_keys) unless all_storage_keys.empty? + end + + RequestStore.delete(:circuitbreaker_cache) + end + + def self.load(cache_key) + first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| + redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) + end + + last_failure = Time.at(last_failure.to_i) if last_failure.present? + first_failure = Time.at(first_failure.to_i) if first_failure.present? + + new(first_failure, last_failure, failure_count.to_i) + end + + def initialize(first_failure, last_failure, failure_count) + @first_failure = first_failure + @last_failure = last_failure + @failure_count = failure_count + end + + def no_failures? + first_failure.blank? && last_failure.blank? && failure_count == 0 + end + end + end + end +end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index a12d52d295f..261c936c689 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -11,6 +11,9 @@ module Gitlab # These will always have nil values attr_reader :storage_path + delegate :last_failure, :failure_count, :no_failures?, + to: :failure_info + def initialize(storage, hostname, error: nil) @storage = storage @hostname = hostname @@ -29,16 +32,17 @@ module Gitlab false end - def last_failure - circuit_broken? ? Time.now : nil - end - - def failure_count - circuit_broken? ? failure_count_threshold : 0 - end - def failure_info - Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + @failure_info ||= + if circuit_broken? + Gitlab::Git::Storage::FailureInfo.new(Time.now, + Time.now, + failure_count_threshold) + else + Gitlab::Git::Storage::FailureInfo.new(nil, + nil, + 0) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8998c4b1a83..56f6febe86d 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -102,18 +102,15 @@ module Gitlab end def check_project_moved! - return unless redirected_path + return if redirected_path.nil? - url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - message = <<-MESSAGE.strip_heredoc - Project '#{redirected_path}' was moved to '#{project.full_path}'. + project_moved = Checks::ProjectMoved.new(project, user, redirected_path, protocol) - Please update your Git remote and try again: - - git remote set-url origin #{url} - MESSAGE - - raise ProjectMovedError, message + if project_moved.permanent_redirect? + project_moved.add_redirect_message + else + raise ProjectMovedError, project_moved.redirect_message(rejected: true) + end end def check_command_disabled!(cmd) @@ -166,7 +163,7 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:cannot_push_to_read_only] + raise UnauthorizedError, push_to_read_only_message end if deploy_key @@ -280,5 +277,9 @@ module Gitlab UserAccess.new(user, project: project) end end + + def push_to_read_only_message + ERROR_MESSAGES[:cannot_push_to_read_only] + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 98f1f45b338..1c9477e84b2 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -19,10 +19,14 @@ module Gitlab end if Gitlab::Database.read_only? - raise UnauthorizedError, ERROR_MESSAGES[:read_only] + raise UnauthorizedError, push_to_read_only_message end true end + + def push_to_read_only_message + ERROR_MESSAGES[:read_only] + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 1fe938a39a8..b753ac46291 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -27,7 +27,7 @@ module Gitlab end SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - MAXIMUM_GITALY_CALLS = 30 + MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index b9e606592d7..a477d618f63 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -87,6 +87,17 @@ module Gitlab response.result end + + def fsck + request = Gitaly::FsckRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :fsck, request) + + if response.error.empty? + return "", 0 + else + return response.error.b, 1 + end + end end end end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 94678b6ec40..3f3f10596c5 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -2,9 +2,8 @@ # key-13 or user-36 or last commit module Gitlab module Identifier - def identify(identifier, project, newrev) + def identify(identifier, project = nil, newrev = nil) if identifier.blank? - # Local push from gitlab identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http @@ -17,6 +16,8 @@ module Gitlab # Tries to identify a user based on a commit SHA. def identify_using_commit(project, ref) + return if project.nil? && ref.nil? + commit = project.commit(ref) return if !commit || !commit.author_email diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index f4f9b5ca792..5a0f7f28fc8 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -27,7 +27,6 @@ module Gitlab def sample sample_memory_usage sample_file_descriptors - sample_objects sample_gc flush @@ -48,29 +47,6 @@ module Gitlab add_metric('file_descriptors', value: System.file_descriptor_count) end - if Metrics.mri? - def sample_objects - sample = Allocations.to_hash - counts = sample.each_with_object({}) do |(klass, count), hash| - name = klass.name - - next unless name - - hash[name] = count - end - - # Symbols aren't allocated so we'll need to add those manually. - counts['Symbol'] = Symbol.all_symbols.length - - counts.each do |name, count| - add_metric('object_counts', { count: count }, type: name) - end - end - else - def sample_objects - end - end - def sample_gc time = GC::Profiler.total_time * 1000.0 stats = GC.stat.merge(total_time: time) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 436a9e9550d..b68800417a2 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -32,7 +32,7 @@ module Gitlab def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', {}) + metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) GC.stat.keys.each do |key| metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) @@ -48,7 +48,6 @@ module Gitlab def sample start_time = System.monotonic_time sample_gc - sample_objects metrics[:memory_usage].set(labels, System.memory_usage) metrics[:file_descriptors].set(labels, System.file_descriptor_count) @@ -68,41 +67,15 @@ module Gitlab end end - def sample_objects - list_objects.each do |name, count| - metrics[:objects_total].set(labels.merge(class: name), count) - end - end - - if Metrics.mri? - def list_objects - sample = Allocations.to_hash - counts = sample.each_with_object({}) do |(klass, count), hash| - name = klass.name - - next unless name - - hash[name] = count - end - - # Symbols aren't allocated so we'll need to add those manually. - counts['Symbol'] = Symbol.all_symbols.length - counts - end - else - def list_objects - end - end - def worker_label return {} unless defined?(Unicorn::Worker) worker_no = ::Prometheus::Client::Support::Unicorn.worker_id if worker_no - { unicorn: worker_no } + { worker: worker_no } else - { unicorn: 'master' } + { worker: 'master' } end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index bc836dcc08d..9ff82d628c0 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,7 +1,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user epic).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb new file mode 100644 index 00000000000..fe81513c9ec --- /dev/null +++ b/lib/gitlab/storage_check.rb @@ -0,0 +1,11 @@ +require_relative 'storage_check/cli' +require_relative 'storage_check/gitlab_caller' +require_relative 'storage_check/option_parser' +require_relative 'storage_check/response' + +module Gitlab + module StorageCheck + ENDPOINT = '/-/storage_check'.freeze + Options = Struct.new(:target, :token, :interval, :dryrun) + end +end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb new file mode 100644 index 00000000000..04bf1bf1d26 --- /dev/null +++ b/lib/gitlab/storage_check/cli.rb @@ -0,0 +1,69 @@ +module Gitlab + module StorageCheck + class CLI + def self.start!(args) + runner = new(Gitlab::StorageCheck::OptionParser.parse!(args)) + runner.start_loop + end + + attr_reader :logger, :options + + def initialize(options) + @options = options + @logger = Logger.new(STDOUT) + end + + def start_loop + logger.info "Checking #{options.target} every #{options.interval} seconds" + + if options.dryrun + logger.info "Dryrun, exiting..." + return + end + + begin + loop do + response = GitlabCaller.new(options).call! + log_response(response) + update_settings(response) + + sleep options.interval + end + rescue Interrupt + logger.info "Ending storage-check" + end + end + + def update_settings(response) + previous_interval = options.interval + + if response.valid? + options.interval = response.check_interval || previous_interval + end + + if previous_interval != options.interval + logger.info "Interval changed: #{options.interval} seconds" + end + end + + def log_response(response) + unless response.valid? + return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}") + end + + if response.responsive_shards.any? + logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}") + end + + warnings = [] + if response.skipped_shards.any? + warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" + end + if response.failing_shards.any? + warnings << "Failing shards: #{response.failing_shards.join(', ')}" + end + logger.warn(warnings.join(' - ')) if warnings.any? + end + end + end +end diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb new file mode 100644 index 00000000000..44952b68844 --- /dev/null +++ b/lib/gitlab/storage_check/gitlab_caller.rb @@ -0,0 +1,39 @@ +require 'excon' + +module Gitlab + module StorageCheck + class GitlabCaller + def initialize(options) + @options = options + end + + def call! + Gitlab::StorageCheck::Response.new(get_response) + rescue Errno::ECONNREFUSED, Excon::Error + # Server not ready, treated as invalid response. + Gitlab::StorageCheck::Response.new(nil) + end + + def get_response + scheme, *other_parts = URI.split(@options.target) + socket_path = if scheme == 'unix' + other_parts.compact.join + end + + connection = Excon.new(@options.target, socket: socket_path) + connection.post(path: Gitlab::StorageCheck::ENDPOINT, + headers: headers) + end + + def headers + @headers ||= begin + headers = {} + headers['Content-Type'] = headers['Accept'] = 'application/json' + headers['TOKEN'] = @options.token if @options.token + + headers + end + end + end + end +end diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb new file mode 100644 index 00000000000..66ed7906f97 --- /dev/null +++ b/lib/gitlab/storage_check/option_parser.rb @@ -0,0 +1,39 @@ +module Gitlab + module StorageCheck + class OptionParser + def self.parse!(args) + # Start out with some defaults + options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false) + + parser = ::OptionParser.new do |opts| + opts.banner = "Usage: bin/storage_check [options]" + + opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value| + options.target = value + end + + opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value } + + opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value| + options.interval = value + end + + opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value| + options.dryrun = value + end + end + parser.parse!(args) + + unless options.target + raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks') + end + + if URI.parse(options.target).scheme.nil? + raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported') + end + + options + end + end + end +end diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb new file mode 100644 index 00000000000..326ab236e3e --- /dev/null +++ b/lib/gitlab/storage_check/response.rb @@ -0,0 +1,77 @@ +require 'json' + +module Gitlab + module StorageCheck + class Response + attr_reader :http_response + + def initialize(http_response) + @http_response = http_response + end + + def valid? + @http_response && (200...299).cover?(@http_response.status) && + @http_response.headers['Content-Type'].include?('application/json') && + parsed_response + end + + def check_interval + return nil unless parsed_response + + parsed_response['check_interval'] + end + + def responsive_shards + divided_results[:responsive_shards] + end + + def skipped_shards + divided_results[:skipped_shards] + end + + def failing_shards + divided_results[:failing_shards] + end + + private + + def results + return [] unless parsed_response + + parsed_response['results'] + end + + def divided_results + return @divided_results if @divided_results + + @divided_results = {} + @divided_results[:responsive_shards] = [] + @divided_results[:skipped_shards] = [] + @divided_results[:failing_shards] = [] + + results.each do |info| + name = info['storage'] + + case info['success'] + when true + @divided_results[:responsive_shards] << name + when false + @divided_results[:failing_shards] << name + else + @divided_results[:skipped_shards] << name + end + end + + @divided_results + end + + def parsed_response + return @parsed_response if defined?(@parsed_response) + + @parsed_response = JSON.parse(@http_response.body) + rescue JSON::JSONError + @parsed_response = nil + end + end + end +end |