diff options
| author | Mike Greiling <mike@pixelcog.com> | 2017-10-24 07:52:21 +0000 |
|---|---|---|
| committer | Mike Greiling <mike@pixelcog.com> | 2017-10-24 07:52:21 +0000 |
| commit | f7bdfe5098c9a5a3a426344ba1d7dd668212cf7e (patch) | |
| tree | 5a1cb2a90b0a2b5114f6f23712b68724b3e8a7e0 /lib | |
| parent | 9cd528aa7ac5066570cb50b909951a6c6b723935 (diff) | |
| parent | e16add2267648d3d3ef1d98f9b53f67a29428791 (diff) | |
| download | gitlab-ce-f7bdfe5098c9a5a3a426344ba1d7dd668212cf7e.tar.gz | |
Merge branch 'master' into 'es-module-autosave'es-module-autosave
# Conflicts:
# app/assets/javascripts/issuable_form.js
# app/assets/javascripts/notes.js
Diffstat (limited to 'lib')
53 files changed, 1074 insertions, 352 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 79e55a2f4f7..99fcc59ba04 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -4,6 +4,10 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") + NO_SLASH_URL_PART_REGEX = %r{[^/]+} + PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + use GrapeLogging::Middleware::RequestLogger, logger: Logger.new(LOG_FILENAME), formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, @@ -96,9 +100,6 @@ module API helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers - NO_SLASH_URL_PART_REGEX = %r{[^/]+} - PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze - # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 61a2d688282..19152c9f395 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -8,6 +8,16 @@ module API before { authorize! :download_code, user_project } + helpers do + def find_branch!(branch_name) + begin + user_project.repository.find_branch(branch_name) || not_found!('Branch') + rescue Gitlab::Git::CommandError + render_api_error!('The branch refname is invalid', 400) + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -38,8 +48,7 @@ module API user_project.repository.branch_exists?(params[:branch]) ? status(204) : status(404) end get do - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) present branch, with: Entities::Branch, project: user_project end @@ -60,8 +69,7 @@ module API put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) protected_branch = user_project.protected_branches.find_by(name: branch.name) @@ -96,8 +104,7 @@ module API put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless branch + branch = find_branch!(params[:branch]) protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy @@ -133,8 +140,7 @@ module API delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + branch = find_branch!(params[:branch]) commit = user_project.repository.commit(branch.dereferenced_target) diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 4af37a2ad1d..2685dc27252 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -4,8 +4,6 @@ module API class Commits < Grape::API include PaginationParams - COMMIT_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: API::NO_SLASH_URL_PART_REGEX) - before { authorize! :download_code, user_project } params do @@ -85,7 +83,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -99,7 +97,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/diff', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -115,7 +113,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -132,7 +130,7 @@ module API requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' requires :branch, type: String, desc: 'The name of the branch' end - post ':id/repository/commits/:sha/cherry_pick', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project commit = user_project.commit(params[:sha]) @@ -169,7 +167,7 @@ module API requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end - post ':id/repository/commits/:sha/comments', requirements: COMMIT_ENDPOINT_REQUIREMENTS do + post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -186,7 +184,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index ceee3226732..7887b886c03 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,7 +57,7 @@ module API desc 'Get raw blob contents from the repository' params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha/raw' do assign_blob_vars! @@ -67,7 +67,7 @@ module API desc 'Get a blob from the repository' params do - requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha' do assign_blob_vars! diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index c189d486f50..f493fd7c7ec 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 345cb7e7c11..ed206a6def0 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -11,7 +11,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do desc 'Get a project repository commits' do success ::API::Entities::Commit end @@ -72,7 +72,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha" do + get ":id/repository/commits/:sha", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit @@ -86,7 +86,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ":id/repository/commits/:sha/diff" do + get ":id/repository/commits/:sha/diff", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit @@ -102,7 +102,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - get ':id/repository/commits/:sha/comments' do + get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -119,7 +119,7 @@ module API requires :sha, type: String, desc: 'A commit sha to be cherry picked' requires :branch, type: String, desc: 'The name of the branch' end - post ':id/repository/commits/:sha/cherry_pick' do + post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize! :push_code, user_project commit = user_project.commit(params[:sha]) @@ -156,7 +156,7 @@ module API requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end - post ':id/repository/commits/:sha/comments' do + post ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -173,7 +173,7 @@ module API lines.each do |line| next unless line.new_pos == params[:line] && line.type == params[:line_type] - break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + break opts[:line_code] = Gitlab::Git.diff_line_code(diff.new_path, line.new_pos, line.old_pos) end break if opts[:line_code] diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index 41a7c6b83ae..f9a47101e27 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: %r{[^/]+} } do + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -43,7 +43,7 @@ module API requires :sha, type: String, desc: 'The commit, branch name, or tag name' requires :filepath, type: String, desc: 'The path to the file to display' end - get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do + get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"], requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do repo = user_project.repository commit = repo.commit(params[:sha]) not_found! "Commit" unless commit @@ -56,7 +56,7 @@ module API params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' end - get ':id/repository/raw_blobs/:sha' do + get ':id/repository/raw_blobs/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do repo = user_project.repository begin blob = Gitlab::Git::Blob.raw(repo, params[:sha]) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 3cf3939994a..05aa79dc160 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -101,50 +101,52 @@ module Backup end def unpack - Dir.chdir(backup_path) - - # check for existing backups in the backup dir - if backup_file_list.empty? - $progress.puts "No backups found in #{backup_path}" - $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" - exit 1 - elsif backup_file_list.many? && ENV["BACKUP"].nil? - $progress.puts 'Found more than one backup, please specify which one you want to restore:' - $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' - exit 1 - end + Dir.chdir(backup_path) do + # check for existing backups in the backup dir + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" + $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + exit 1 + elsif backup_file_list.many? && ENV["BACKUP"].nil? + $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + exit 1 + end - tar_file = if ENV['BACKUP'].present? - "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" - else - backup_file_list.first - end + tar_file = if ENV['BACKUP'].present? + "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + backup_file_list.first + end - unless File.exist?(tar_file) - $progress.puts "The backup file #{tar_file} does not exist!" - exit 1 - end + unless File.exist?(tar_file) + $progress.puts "The backup file #{tar_file} does not exist!" + exit 1 + end - $progress.print 'Unpacking backup ... ' + $progress.print 'Unpacking backup ... ' - unless Kernel.system(*%W(tar -xf #{tar_file})) - $progress.puts 'unpacking backup failed'.color(:red) - exit 1 - else - $progress.puts 'done'.color(:green) - end + unless Kernel.system(*%W(tar -xf #{tar_file})) + $progress.puts 'unpacking backup failed'.color(:red) + exit 1 + else + $progress.puts 'done'.color(:green) + end - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != Gitlab::VERSION - $progress.puts 'GitLab version mismatch:'.color(:red) - $progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) - $progress.puts ' Please switch to the following version and try again:'.color(:red) - $progress.puts " version: #{settings[:gitlab_version]}".color(:red) - $progress.puts - $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" - exit 1 + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != Gitlab::VERSION + $progress.puts(<<~HEREDOC.color(:red)) + GitLab version mismatch: + Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! + Please switch to the following version and try again: + version: #{settings[:gitlab_version]} + HEREDOC + $progress.puts + $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + exit 1 + end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d8c8deea628..6786b9d07b6 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -75,9 +75,19 @@ module Banzai begin node['href'] = node['href'].strip uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + return unless uri.scheme + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + scheme = uri.scheme + .strip + .downcase + .gsub(/[^A-Za-z0-9\+\.\-]+/, '') + + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) rescue Addressable::URI::InvalidURIError node.remove_attribute('href') end diff --git a/lib/github/import.rb b/lib/github/import.rb index 55f8387f27a..76612799412 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -74,7 +74,7 @@ module Github def fetch_wiki_repository return if project.wiki.repository_exists? - wiki_path = "#{project.disk_path}.wiki" + wiki_path = project.wiki.disk_path gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 1b5be91461b..83bf0b5310d 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -23,7 +23,7 @@ module Github private def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index 8e5c95f2287..380802258f5 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -81,6 +81,7 @@ module Gitlab def single_diff_rows(merge_request_diff) sha_attribute = Gitlab::Database::ShaAttribute.new commits = YAML.load(merge_request_diff.st_commits) rescue [] + commits ||= [] commit_rows = commits.map.with_index do |commit, index| commit_hash = commit.to_hash.with_indifferent_access.except(:parent_ids) diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d1979bb7ed3..033ecd15749 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -241,7 +241,7 @@ module Gitlab end def generate_line_code(pr_comment) - Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + Gitlab::Git.diff_line_code(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) end def pull_request_comment_attributes(comment) diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 98dfe900044..2a0cb640a14 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,82 +4,29 @@ module Gitlab include Gitlab::Routing include IconsHelper - MissingResolution = Class.new(ResolutionError) - CONTEXT_LINES = 3 - attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository - - def initialize(merge_file_result, conflict, merge_request:) - @merge_file_result = merge_file_result - @their_path = conflict[:theirs][:path] - @our_path = conflict[:ours][:path] - @our_mode = conflict[:ours][:mode] - @merge_request = merge_request - @repository = merge_request.project.repository - @match_line_headers = {} - end - - def content - merge_file_result[:data] - end + attr_reader :merge_request - def our_blob - @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) - end + # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps + attr_reader :raw - def type - lines unless @type + delegate :type, :content, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw - @type.inquiry + def initialize(raw, merge_request:) + @raw = raw + @merge_request = merge_request + @match_line_headers = {} end - # Array of Gitlab::Diff::Line objects def lines return @lines if defined?(@lines) - begin - @type = 'text' - @lines = Gitlab::Conflict::Parser.new.parse(content, - our_path: our_path, - their_path: their_path, - parent_file: self) - rescue Gitlab::Conflict::Parser::ParserError - @type = 'text-editor' - @lines = nil - end + @lines = raw.lines.nil? ? nil : map_raw_lines(raw.lines) end def resolve_lines(resolution) - section_id = nil - - lines.map do |line| - unless line.type - section_id = nil - next line - end - - section_id ||= line_code(line) - - case resolution[section_id] - when 'head' - next unless line.type == 'new' - when 'origin' - next unless line.type == 'old' - else - raise MissingResolution, "Missing resolution for section ID: #{section_id}" - end - - line - end.compact - end - - def resolve_content(resolution) - if resolution == content - raise MissingResolution, "Resolved content has no changes for file #{our_path}" - end - - resolution + map_raw_lines(raw.resolve_lines(resolution)) end def highlight_lines! @@ -163,7 +110,7 @@ module Gitlab end def line_code(line) - Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(our_path, line.new_pos, line.old_pos) end def create_match_line(line) @@ -227,15 +174,14 @@ module Gitlab new_path: our_path) end - # Don't try to print merge_request or repository. - def inspect - instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| - value = instance_variable_get("@#{instance_variable}") + private - "#{instance_variable}=\"#{value}\"" + def map_raw_lines(raw_lines) + raw_lines.map do |raw_line| + Gitlab::Diff::Line.new(raw_line[:full_line], raw_line[:type], + raw_line[:line_obj_index], raw_line[:line_old], + raw_line[:line_new], parent_file: self) end - - "#<#{self.class} #{instance_variables.join(' ')}>" end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 90f83e0f810..fb28e80ff73 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,48 +1,29 @@ module Gitlab module Conflict class FileCollection - ConflictSideMissing = Class.new(StandardError) - - attr_reader :merge_request, :our_commit, :their_commit, :project - - delegate :repository, to: :project - - class << self - # We can only write when getting the merge index from the source - # project, because we will write to that project. We don't use this all - # the time because this fetches a ref into the source project, which - # isn't needed for reading. - def for_resolution(merge_request) - project = merge_request.source_project - - new(merge_request, project).tap do |file_collection| - project - .repository - .with_repo_branch_commit(merge_request.target_project.repository.raw_repository, merge_request.target_branch) do - - yield file_collection - end - end - end - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - def read_only(merge_request) - new(merge_request, merge_request.target_project) - end + attr_reader :merge_request, :resolver + + def initialize(merge_request) + source_repo = merge_request.source_project.repository.raw + our_commit = merge_request.source_branch_head.raw + their_commit = merge_request.target_branch_head.raw + target_repo = merge_request.target_project.repository.raw + @resolver = Gitlab::Git::Conflict::Resolver.new(source_repo, our_commit, target_repo, their_commit) + @merge_request = merge_request end - def merge_index - @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + def resolve(user, commit_message, files) + args = { + source_branch: merge_request.source_branch, + target_branch: merge_request.target_branch, + commit_message: commit_message || default_commit_message + } + resolver.resolve_conflicts(user, files, args) end def files - @files ||= merge_index.conflicts.map do |conflict| - raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] - - Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), - conflict, - merge_request: merge_request) + @files ||= resolver.conflicts.map do |conflict_file| + Gitlab::Conflict::File.new(conflict_file, merge_request: merge_request) end end @@ -61,8 +42,8 @@ module Gitlab end def default_commit_message - conflict_filenames = merge_index.conflicts.map do |conflict| - "# #{conflict[:ours][:path]}" + conflict_filenames = files.map do |conflict| + "# #{conflict.our_path}" end <<EOM.chomp @@ -72,15 +53,6 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end - - private - - def initialize(merge_request, project) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.rugged_commit - @their_commit = merge_request.target_branch_head.raw.rugged_commit - @project = project - end end end end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb deleted file mode 100644 index e3678c914db..00000000000 --- a/lib/gitlab/conflict/parser.rb +++ /dev/null @@ -1,74 +0,0 @@ -module Gitlab - module Conflict - class Parser - UnresolvableError = Class.new(StandardError) - UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) - - # Recoverable errors - the conflict can be resolved in an editor, but not with - # sections. - ParserError = Class.new(StandardError) - UnexpectedDelimiter = Class.new(ParserError) - MissingEndDelimiter = Class.new(ParserError) - - def parse(text, our_path:, their_path:, parent_file: nil) - validate_text!(text) - - line_obj_index = 0 - line_old = 1 - line_new = 1 - type = nil - lines = [] - conflict_start = "<<<<<<< #{our_path}" - conflict_middle = '=======' - conflict_end = ">>>>>>> #{their_path}" - - text.each_line.map do |line| - full_line = line.delete("\n") - - if full_line == conflict_start - validate_delimiter!(type.nil?) - - type = 'new' - elsif full_line == conflict_middle - validate_delimiter!(type == 'new') - - type = 'old' - elsif full_line == conflict_end - validate_delimiter!(type == 'old') - - type = nil - elsif line[0] == '\\' - type = 'nonewline' - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - else - lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) - line_old += 1 if type != 'new' - line_new += 1 if type != 'old' - - line_obj_index += 1 - end - end - - raise MissingEndDelimiter unless type.nil? - - lines - end - - private - - def validate_text!(text) - raise UnmergeableFile if text.blank? # Typically a binary file - raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? - end - - def validate_delimiter!(condition) - raise UnexpectedDelimiter unless condition - end - end - end -end diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb deleted file mode 100644 index 0b61256b35a..00000000000 --- a/lib/gitlab/conflict/resolution_error.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module Conflict - ResolutionError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 599c3c5deab..ea5891a028a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -49,7 +49,7 @@ module Gitlab def line_code(line) return if line.meta? - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def line_for_line_code(code) diff --git a/lib/gitlab/diff/line_code.rb b/lib/gitlab/diff/line_code.rb deleted file mode 100644 index f3578ab3d35..00000000000 --- a/lib/gitlab/diff/line_code.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Diff - class LineCode - def self.generate(file_path, new_line_position, old_line_position) - "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" - end - end - end -end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 742f989c50b..7dc9cc7c281 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -17,7 +17,9 @@ module Gitlab # without having to instantiate all the others that come after it. Enumerator.new do |yielder| @lines.each do |line| - next if filename?(line) + # We're expecting a filename parameter only in a meta-part of the diff content + # when type is defined then we're already in a content-part + next if filename?(line) && type.nil? full_line = line.delete("\n") diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 7b3483a7f96..99dfee3dd9b 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,9 +14,9 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to? :force_encoding + return nil unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - # if message is utf-8 encoding, just return it message.force_encoding("UTF-8") return message if message.valid_encoding? @@ -50,6 +50,9 @@ module Gitlab end def encode_utf8(message) + return nil unless message.is_a?(String) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] begin diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index a8cb7fc3fe7..0e9ef4f897c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,31 +6,33 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme/i, - changelog: /\A(changelog|history|changes|news)/i, - license: /\A(licen[sc]e|copying)(\..+|\z)/i, - contributing: /\Acontributing/i, + readme: /\Areadme[^\/]*\z/i, + changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, + license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, + contributing: /\Acontributing[^\/]*\z/i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, + issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, + merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - route_map: 'route-map.yml', + route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile/, + cartfile: /\ACartfile[^\/]*\z/, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\.gemspec\z/, + gemspec: /\A[^\/]*\.gemspec\z/, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\.podspec\.json\z/, - podspec: /\.podspec\z/, - requirements_txt: /requirements\.txt\z/, + podspec_json: /\A[^\/]*\.podspec\.json\z/, + podspec: /\A[^\/]*\.podspec\z/, + requirements_txt: /\A[^\/]*requirements\.txt\z/, yarn_lock: 'yarn.lock' }.freeze @@ -63,13 +65,11 @@ module Gitlab # type_of('README.md') # => :readme # type_of('VERSION') # => :version def self.type_of(path) - name = File.basename(path) - PATTERNS.each do |type, search| did_match = if search.is_a?(Regexp) - name =~ search + path =~ search else - name.casecmp(search) == 0 + path.casecmp(search) == 0 end return type if did_match diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index c78fe63f9b5..1f31cdbc96d 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -66,6 +66,10 @@ module Gitlab end end end + + def diff_line_code(file_path, new_line_position, old_line_position) + "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" + end end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb new file mode 100644 index 00000000000..fc1595f1faf --- /dev/null +++ b/lib/gitlab/git/conflict/file.rb @@ -0,0 +1,86 @@ +module Gitlab + module Git + module Conflict + class File + attr_reader :content, :their_path, :our_path, :our_mode, :repository + + def initialize(repository, commit_oid, conflict, content) + @repository = repository + @commit_oid = commit_oid + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @content = content + end + + def lines + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Git::Conflict::Parser.parse(content, + our_path: our_path, + their_path: their_path) + rescue Gitlab::Git::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end + end + + def type + lines unless @type + + @type.inquiry + end + + def our_blob + # REFACTOR NOTE: the source of `commit_oid` used to be + # `merge_request.diff_refs.head_sha`. Instead of passing this value + # around the new lib structure, I decided to use `@commit_oid` which is + # equivalent to `merge_request.source_branch_head.raw.rugged_commit.oid`. + # That is what `merge_request.diff_refs.head_sha` is equivalent to when + # `merge_request` is not persisted (see `MergeRequest#diff_head_commit`). + # I think using the same oid is more consistent anyways, but if Conflicts + # start breaking, the change described above is a good place to look at. + @our_blob ||= repository.blob_at(@commit_oid, our_path) + end + + def line_code(line) + Gitlab::Git.diff_line_code(our_path, line[:line_new], line[:line_old]) + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line[:type] + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line[:type] == 'new' + when 'origin' + next unless line[:type] == 'old' + else + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def resolve_content(resolution) + if resolution == content + raise Gitlab::Git::Conflict::Resolver::ResolutionError, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb new file mode 100644 index 00000000000..3effa9d2d31 --- /dev/null +++ b/lib/gitlab/git/conflict/parser.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Parser + UnresolvableError = Class.new(StandardError) + UnmergeableFile = Class.new(UnresolvableError) + UnsupportedEncoding = Class.new(UnresolvableError) + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + ParserError = Class.new(StandardError) + UnexpectedDelimiter = Class.new(ParserError) + MissingEndDelimiter = Class.new(ParserError) + + class << self + def parse(text, our_path:, their_path:, parent_file: nil) + validate_text!(text) + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + validate_delimiter!(type.nil?) + + type = 'new' + elsif full_line == conflict_middle + validate_delimiter!(type == 'new') + + type = 'old' + elsif full_line == conflict_end + validate_delimiter!(type == 'old') + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + else + lines << { + full_line: full_line, + type: type, + line_obj_index: line_obj_index, + line_old: line_old, + line_new: line_new + } + + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + + private + + def validate_text!(text) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? + end + + def validate_delimiter!(condition) + raise UnexpectedDelimiter unless condition + end + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb new file mode 100644 index 00000000000..df509c5f4ce --- /dev/null +++ b/lib/gitlab/git/conflict/resolver.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Conflict + class Resolver + ConflictSideMissing = Class.new(StandardError) + ResolutionError = Class.new(StandardError) + + def initialize(repository, our_commit, target_repository, their_commit) + @repository = repository + @our_commit = our_commit.rugged_commit + @target_repository = target_repository + @their_commit = their_commit.rugged_commit + end + + def conflicts + @conflicts ||= begin + target_index = @target_repository.rugged.merge_commits(@our_commit, @their_commit) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + target_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Git::Conflict::File.new( + @target_repository, + @our_commit.oid, + conflict, + target_index.merge_file(conflict[:ours][:path])[:data] + ) + end + end + end + + def resolve_conflicts(user, files, source_branch:, target_branch:, commit_message:) + @repository.with_repo_branch_commit(@target_repository, target_branch) do + files.each do |file_params| + conflict_file = conflict_for_path(file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: commit_message, + parents: [@our_commit, @their_commit].map(&:oid) + } + + @repository.commit_index(user, source_branch, index, commit_params) + end + end + + def conflict_for_path(old_path, new_path) + conflicts.find do |conflict| + conflict.their_path == old_path && conflict.our_path == new_path + end + end + + private + + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def index + @index ||= @repository.rugged.merge_commits(@our_commit, @their_commit) + end + + def write_resolved_file_to_index(file, params) + if params[:sections] + resolved_lines = file.resolve_lines(params[:sections]) + new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") + + new_file << "\n" if file.our_blob.data.ends_with?("\n") + elsif params[:content] + new_file = file.resolve_content(params[:content]) + end + + our_path = file.our_path + + index.add(path: our_path, oid: @repository.rugged.write(new_file, :blob), mode: file.our_mode) + index.conflict_remove(our_path) + end + end + end + end +end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 80f0731cd99..9d0b47a1a6d 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -30,6 +30,17 @@ module Gitlab RequestStore.fetch(:gitlab_git_env) { {} } end + def self.to_env_hash + env = {} + + all.compact.each do |key, value| + value = value.join(File::PATH_SEPARATOR) if value.is_a?(Array) + env[key.to_s] = value + end + + env + end + def self.[](key) all[key] end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index d835dcca8ba..ab94ba8a73a 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -3,9 +3,17 @@ module Gitlab class OperationService include Gitlab::Git::Popen - WithBranchResult = Struct.new(:newrev, :repo_created, :branch_created) do + BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created + + def self.from_gitaly(branch_update) + new( + branch_update.commit_id, + branch_update.repo_created, + branch_update.branch_created + ) + end end attr_reader :user, :repository @@ -112,7 +120,7 @@ module Gitlab ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name update_ref_in_hooks(ref, newrev, oldrev) - WithBranchResult.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) + BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) end def find_oldrev_from_branch(newrev, branch) diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 3d2fc471d28..b45da6020ee 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -5,6 +5,8 @@ require 'open3' module Gitlab module Git module Popen + FAST_GIT_PROCESS_TIMEOUT = 15.seconds + def popen(cmd, path, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" @@ -27,6 +29,67 @@ module Gitlab [@cmd_output, @cmd_status] end + + def popen_with_timeout(cmd, timeout, path, vars = {}) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + path ||= Dir.pwd + vars['PWD'] = path + + unless File.directory?(path) + FileUtils.mkdir_p(path) + end + + rout, wout = IO.pipe + rerr, werr = IO.pipe + + pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) + + begin + status = process_wait_with_timeout(pid, timeout) + + # close write ends so we could read them + wout.close + werr.close + + cmd_output = rout.readlines.join + cmd_output << rerr.readlines.join # Copying the behaviour of `popen` which merges stderr into output + + [cmd_output, status.exitstatus] + rescue Timeout::Error => e + kill_process_group_for_pid(pid) + + raise e + ensure + wout.close unless wout.closed? + werr.close unless werr.closed? + + rout.close + rerr.close + end + end + + def process_wait_with_timeout(pid, timeout) + deadline = timeout.seconds.from_now + wait_time = 0.01 + + while deadline > Time.now + sleep(wait_time) + _, status = Process.wait2(pid, Process::WNOHANG) + + return status unless status.nil? + end + + raise Timeout::Error, "Timeout waiting for process ##{pid}" + end + + def kill_process_group_for_pid(pid) + Process.kill("KILL", -pid) + Process.wait(pid) + rescue Errno::ESRCH + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a082cfed706..59a54b48ed9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -704,7 +704,17 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge(user, source_sha, target_branch, message) + def merge(user, source_sha, target_branch, message, &block) + gitaly_migrate(:operation_user_merge_branch) do |is_enabled| + if is_enabled + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) + else + rugged_merge(user, source_sha, target_branch, message, &block) + end + end + end + + def rugged_merge(user, source_sha, target_branch, message) committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) OperationService.new(user, self).with_branch(target_branch) do |start_commit| @@ -1062,6 +1072,13 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb + def run_git_with_timeout(args, timeout, env: {}) + circuit_breaker.perform do + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) + end + end + + # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) end @@ -1086,6 +1103,30 @@ module Gitlab @has_visible_content = has_local_branches? end + def fetch(remote = 'origin') + args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) + + popen(args, @path).last.zero? + end + + def blob_at(sha, path) + Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) + end + + def commit_index(user, branch_name, index, options) + committer = user_to_committer(user) + + OperationService.new(user, self).with_branch(branch_name) do + commit_params = options.merge( + tree: index.write_tree(rugged), + author: committer, + committer: committer + ) + + create_commit(commit_params) + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1230,6 +1271,7 @@ module Gitlab relative_paths.map { |d| File.join(path, d) } else Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES) + .flatten .compact .flat_map { |d| d.split(File::PATH_SEPARATOR) } end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 92a6a672534..60b2a4ec411 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -28,7 +28,7 @@ module Gitlab private def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) + output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) unless status.zero? raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 1eaa2d83fb6..0456ad9a1f3 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -2,15 +2,13 @@ module Gitlab module Git module Storage class CircuitBreaker + include CircuitBreakerSettings + FailureInfo = Struct.new(:last_failure, :failure_count) attr_reader :storage, :hostname, - :storage_path, - :failure_count_threshold, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + :storage_path delegate :last_failure, :failure_count, to: :failure_info @@ -18,7 +16,7 @@ module Gitlab pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.keys(pattern) + all_storage_keys = redis.scan_each(match: pattern).to_a redis.del(*all_storage_keys) unless all_storage_keys.empty? end @@ -53,10 +51,6 @@ module Gitlab config = Gitlab.config.repositories.storages[@storage] @storage_path = config['path'] - @failure_count_threshold = config['failure_count_threshold'] - @failure_wait_time = config['failure_wait_time'] - @failure_reset_time = config['failure_reset_time'] - @storage_timeout = config['storage_timeout'] end def perform diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb new file mode 100644 index 00000000000..d2313fe7c1b --- /dev/null +++ b/lib/gitlab/git/storage/circuit_breaker_settings.rb @@ -0,0 +1,29 @@ +module Gitlab + module Git + module Storage + module CircuitBreakerSettings + def failure_count_threshold + application_settings.circuitbreaker_failure_count_threshold + end + + def failure_wait_time + application_settings.circuitbreaker_failure_wait_time + end + + def failure_reset_time + application_settings.circuitbreaker_failure_reset_time + end + + def storage_timeout + application_settings.circuitbreaker_storage_timeout + end + + private + + def application_settings + Gitlab::CurrentSettings.current_application_settings + end + end + end + end +end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb index 1564e94b7f7..7049772fe3b 100644 --- a/lib/gitlab/git/storage/health.rb +++ b/lib/gitlab/git/storage/health.rb @@ -23,26 +23,36 @@ module Gitlab end end - def self.all_keys_for_storages(storage_names, redis) + private_class_method def self.all_keys_for_storages(storage_names, redis) keys_per_storage = {} redis.pipelined do storage_names.each do |storage_name| pattern = pattern_for_storage(storage_name) + matched_keys = redis.scan_each(match: pattern) - keys_per_storage[storage_name] = redis.keys(pattern) + keys_per_storage[storage_name] = matched_keys end end - keys_per_storage + # We need to make sure each lazy-loaded `Enumerator` for matched keys + # is loaded into an array. + # + # Otherwise it would be loaded in the second `Redis#pipelined` block + # within `.load_for_keys`. In this pipelined call, the active + # Redis-client changes again, so the values would not be available + # until the end of that pipelined-block. + keys_per_storage.each do |storage_name, key_future| + keys_per_storage[storage_name] = key_future.to_a + end end - def self.load_for_keys(keys_per_storage, redis) + private_class_method def self.load_for_keys(keys_per_storage, redis) info_for_keys = {} redis.pipelined do keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.value.map do |key| + info_for_storage = keys_future.map do |key| { name: key, failure_count: redis.hget(key, :failure_count) } end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb index 297c043d054..60c6791a7e4 100644 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ b/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -2,15 +2,14 @@ module Gitlab module Git module Storage class NullCircuitBreaker + include CircuitBreakerSettings + # These will have actual values attr_reader :storage, :hostname # These will always have nil values - attr_reader :storage_path, - :failure_wait_time, - :failure_reset_time, - :storage_timeout + attr_reader :storage_path def initialize(storage, hostname, error: nil) @storage = storage @@ -26,16 +25,12 @@ module Gitlab !!@error end - def failure_count_threshold - 1 - end - def last_failure circuit_broken? ? Time.now : nil end def failure_count - circuit_broken? ? 1 : 0 + circuit_broken? ? failure_count_threshold : 0 end def failure_info diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index d651c931a38..e7b2f52a552 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -23,14 +23,14 @@ module Gitlab end def write_page(name, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - gollum_wiki.write_page(name, format, content, commit_details.to_h) - - nil - rescue Gollum::DuplicatePageError => e - raise Gitlab::Git::Wiki::DuplicatePageError, e.message + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| + if is_enabled + gitaly_write_page(name, format, content, commit_details) + gollum_wiki.clear_cache + else + gollum_write_page(name, format, content, commit_details) + end + end end def delete_page(page_path, commit_details) @@ -110,6 +110,25 @@ module Gitlab raise ArgumentError, "expected a #{klass}, got #{object.inspect}" end end + + def gitaly_wiki_client + @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) + end + + def gollum_write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def gitaly_write_page(name, format, content, commit_details) + gitaly_wiki_client.write_page(name, format, content, commit_details) + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index cf36106e23d..6868be26758 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -33,6 +33,12 @@ module Gitlab MUTEX = Mutex.new private_constant :MUTEX + class << self + attr_accessor :query_time + end + + self.query_time = 0 + def self.stub(name, storage) MUTEX.synchronize do @stubs ||= {} @@ -70,15 +76,30 @@ module Gitlab # All Gitaly RPC call sites should use GitalyClient.call. This method # makes sure that per-request authentication headers are set. + # + # This method optionally takes a block which receives the keyword + # arguments hash 'kwargs' that will be passed to gRPC. This allows the + # caller to modify or augment the keyword arguments. The block must + # return a hash. + # + # For example: + # + # GitalyClient.call(storage, service, rpc, request) do |kwargs| + # kwargs.merge(deadline: Time.now + 10) + # end + # def self.call(storage, service, rpc, request) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) enforce_gitaly_request_limits(:call) - metadata = request_metadata(storage) - metadata = yield(metadata) if block_given? - stub(service, storage).__send__(rpc, request, metadata) # rubocop:disable GitlabSecurity/PublicSend + kwargs = request_kwargs(storage) + kwargs = yield(kwargs) if block_given? + stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend + ensure + self.query_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start end - def self.request_metadata(storage) + def self.request_kwargs(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { 'authorization' => "Bearer #{encoded_token}", diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 81ddaf13e10..91f34011f6e 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -74,6 +74,37 @@ module Gitlab raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error end end + + def user_merge_branch(user, source_sha, target_branch, message) + request_enum = QueueEnumerator.new + response_enum = GitalyClient.call( + @repository.storage, + :operation_service, + :user_merge_branch, + request_enum.each + ) + + request_enum.push( + Gitaly::UserMergeBranchRequest.new( + repository: @gitaly_repo, + user: Util.gitaly_user(user), + commit_id: source_sha, + branch: GitalyClient.encode(target_branch), + message: GitalyClient.encode(message) + ) + ) + + yield response_enum.next.commit_id + + request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) + + branch_update = response_enum.next.branch_update + raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + ensure + request_enum.close + end end end end diff --git a/lib/gitlab/gitaly_client/queue_enumerator.rb b/lib/gitlab/gitaly_client/queue_enumerator.rb new file mode 100644 index 00000000000..b8018029552 --- /dev/null +++ b/lib/gitlab/gitaly_client/queue_enumerator.rb @@ -0,0 +1,28 @@ +module Gitlab + module GitalyClient + class QueueEnumerator + def initialize + @queue = Queue.new + end + + def push(elem) + @queue << elem + end + + def close + push(:close) + end + + def each + return enum_for(:each) unless block_given? + + loop do + elem = @queue.pop + break if elem == :close + + yield elem + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb new file mode 100644 index 00000000000..03afcce81f0 --- /dev/null +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -0,0 +1,45 @@ +require 'stringio' + +module Gitlab + module GitalyClient + class WikiService + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def write_page(name, format, content, commit_details) + request = Gitaly::WikiWritePageRequest.new( + repository: @gitaly_repo, + name: GitalyClient.encode(name), + format: format.to_s, + commit_details: Gitaly::WikiCommitDetails.new( + name: GitalyClient.encode(commit_details.name), + email: GitalyClient.encode(commit_details.email), + message: GitalyClient.encode(commit_details.message) + ) + ) + + strio = StringIO.new(content) + + enum = Enumerator.new do |y| + until strio.eof? + chunk = strio.read(MAX_MSG_SIZE) + request.content = GitalyClient.encode(chunk) + + y.yield request + + request = Gitaly::WikiWritePageRequest.new + end + end + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum) + if error = response.duplicate_error.presence + raise Gitlab::Git::Wiki::DuplicatePageError, error + end + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index e21922070c1..8911b81ec9a 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -38,7 +38,7 @@ module Gitlab end def generate_line_code(line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + Gitlab::Git.diff_line_code(file_path, line.new_pos, line.old_pos) end def on_diff? diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 0396122eeb9..ca8d96f5650 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -8,7 +8,7 @@ module Gitlab end def disk_path - "#{project.disk_path}.wiki" + project.wiki.disk_path end def import_url diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index 635f52131f9..42ded7c286f 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -17,12 +17,32 @@ module Gitlab @model = ancestors_base.model end + # Returns the set of descendants of a given relation, but excluding the given + # relation + def descendants + base_and_descendants.where.not(id: descendants_base.select(:id)) + end + + # Returns the set of ancestors of a given relation, but excluding the given + # relation + # + # Passing an `upto` will stop the recursion once the specified parent_id is + # reached. So all ancestors *lower* than the specified ancestor will be + # included. + def ancestors(upto: nil) + base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) + end + # Returns a relation that includes the ancestors_base set of groups # and all their ancestors (recursively). - def base_and_ancestors + # + # Passing an `upto` will stop the recursion once the specified parent_id is + # reached. So all ancestors *lower* than the specified acestor will be + # included. + def base_and_ancestors(upto: nil) return ancestors_base unless Group.supports_nested_groups? - read_only(base_and_ancestors_cte.apply_to(model.all)) + read_only(base_and_ancestors_cte(upto).apply_to(model.all)) end # Returns a relation that includes the descendants_base set of groups @@ -78,17 +98,19 @@ module Gitlab private - def base_and_ancestors_cte + def base_and_ancestors_cte(stop_id = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) cte << ancestors_base.except(:order) # Recursively get all the ancestors of the base set. - cte << model + parent_query = model .from([groups_table, cte.table]) .where(groups_table[:id].eq(cte.table[:parent_id])) .except(:order) + parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id + cte << parent_query cte end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb new file mode 100644 index 00000000000..eb3c9002710 --- /dev/null +++ b/lib/gitlab/multi_collection_paginator.rb @@ -0,0 +1,61 @@ +module Gitlab + class MultiCollectionPaginator + attr_reader :first_collection, :second_collection, :per_page + + def initialize(*collections, per_page: nil) + raise ArgumentError.new('Only 2 collections are supported') if collections.size != 2 + + @per_page = per_page || Kaminari.config.default_per_page + @first_collection, @second_collection = collections + end + + def paginate(page) + page = page.to_i + paginated_first_collection(page) + paginated_second_collection(page) + end + + def total_count + @total_count ||= first_collection.size + second_collection.size + end + + private + + def paginated_first_collection(page) + @first_collection_pages ||= Hash.new do |hash, page| + hash[page] = first_collection.page(page).per(per_page) + end + + @first_collection_pages[page] + end + + def paginated_second_collection(page) + @second_collection_pages ||= Hash.new do |hash, page| + second_collection_page = page - first_collection_page_count + + offset = if second_collection_page < 1 || first_collection_page_count.zero? + 0 + else + per_page - first_collection_last_page_size + end + hash[page] = second_collection.page(second_collection_page) + .per(per_page - paginated_first_collection(page).size) + .padding(offset) + end + + @second_collection_pages[page] + end + + def first_collection_page_count + return @first_collection_page_count if defined?(@first_collection_page_count) + + first_collection_page = paginated_first_collection(0) + @first_collection_page_count = first_collection_page.total_pages + end + + def first_collection_last_page_size + return @first_collection_last_page_size if defined?(@first_collection_last_page_size) + + @first_collection_last_page_size = paginated_first_collection(first_collection_page_count).count + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7c02c9c5c48..22f8dd669d0 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -26,7 +26,6 @@ module Gitlab apple-touch-icon.png assets autocomplete - boards ci dashboard deploy.html @@ -129,7 +128,6 @@ module Gitlab notification_setting pipeline_quota projects - subgroups ].freeze ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb new file mode 100644 index 00000000000..3f52402b31f --- /dev/null +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -0,0 +1,54 @@ +module Gitlab + module QuickActions + # This class takes spend command argument + # and separates date and time from spend command arguments if it present + # example: + # spend_command_time_and_date = "15m 2017-01-02" + # SpendTimeAndDateSeparator.new(spend_command_time_and_date).execute + # => [900, Mon, 02 Jan 2017] + # if date doesn't present return time with current date + # in other cases return nil + class SpendTimeAndDateSeparator + DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + + def initialize(spend_command_arg) + @spend_arg = spend_command_arg + end + + def execute + return if @spend_arg.blank? + return [get_time, DateTime.now.to_date] unless date_present? + return unless valid_date? + + [get_time, get_date] + end + + private + + def get_time + raw_time = @spend_arg.gsub(DATE_REGEX, '') + Gitlab::TimeTrackingFormatter.parse(raw_time) + end + + def get_date + string_date = @spend_arg.match(DATE_REGEX)[0] + Date.parse(string_date) + end + + def date_present? + DATE_REGEX =~ @spend_arg + end + + def valid_date? + string_date = @spend_arg.match(DATE_REGEX)[0] + date = Date.parse(string_date) rescue nil + + date_past_or_today?(date) + end + + def date_past_or_today?(date) + date&.past? || date&.today? + end + end + end +end diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 67a5f368bdb..33d19373098 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -2,7 +2,7 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash def groups - get_raw(Gitlab::Saml::Config.groups) + Array.wrap(get_raw(Gitlab::Saml::Config.groups)) end private diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index a0a2769cf9e..a1f689d94d9 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -51,6 +51,13 @@ module Gitlab self.num_running(job_ids).zero? end + # Returns true if the given job is running + # + # job_id - The Sidekiq job ID to check. + def self.running?(job_id) + num_running([job_id]) > 0 + end + # Returns the number of jobs that are running. # # job_ids - The Sidekiq job IDs to check. diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index f30c771837a..c99b262f1ca 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -26,7 +26,11 @@ module Gitlab @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end - fragments.join("\n#{union_keyword}\n") + if fragments.any? + fragments.join("\n#{union_keyword}\n") + else + 'NULL' + end end def union_keyword diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 3f3ba77d47f..70a403652e7 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -49,6 +49,8 @@ module Gitlab deployments: Deployment.count, environments: ::Environment.count, gcp_clusters: ::Gcp::Cluster.count, + gcp_clusters_enabled: ::Gcp::Cluster.enabled.count, + gcp_clusters_disabled: ::Gcp::Cluster.disabled.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb new file mode 100644 index 00000000000..385141d44d0 --- /dev/null +++ b/lib/gitlab/utils/merge_hash.rb @@ -0,0 +1,117 @@ +module Gitlab + module Utils + module MergeHash + extend self + # Deep merges an array of hashes + # + # [{ hello: ["world"] }, + # { hello: "Everyone" }, + # { hello: { greetings: ['Bonjour', 'Hello', 'Hallo', 'Dzien dobry'] } }, + # "Goodbye", "Hallo"] + # => [ + # { + # hello: + # [ + # "world", + # "Everyone", + # { greetings: ['Bonjour', 'Hello', 'Hallo', 'Dzien dobry'] } + # ] + # }, + # "Goodbye" + # ] + def merge(elements) + merged, *other_elements = elements + + other_elements.each do |element| + merged = merge_hash_tree(merged, element) + end + + merged + end + + # This extracts all keys and values from a hash into an array + # + # { hello: "world", this: { crushes: ["an entire", "hash"] } } + # => [:hello, "world", :this, :crushes, "an entire", "hash"] + def crush(array_or_hash) + if array_or_hash.is_a?(Array) + crush_array(array_or_hash) + else + crush_hash(array_or_hash) + end + end + + private + + def merge_hash_into_array(array, new_hash) + crushed_new_hash = crush_hash(new_hash) + # Merge the hash into an existing element of the array if there is overlap + if mergeable_index = array.index { |element| crushable?(element) && (crush(element) & crushed_new_hash).any? } + array[mergeable_index] = merge_hash_tree(array[mergeable_index], new_hash) + else + array << new_hash + end + + array + end + + def merge_hash_tree(first_element, second_element) + # If one of the elements is an object, and the other is a Hash or Array + # we can check if the object is already included. If so, we don't need to do anything + # + # Handled cases + # [Hash, Object], [Array, Object] + if crushable?(first_element) && crush(first_element).include?(second_element) + first_element + elsif crushable?(second_element) && crush(second_element).include?(first_element) + second_element + # When the first is an array, we need to go over every element to see if + # we can merge deeper. If no match is found, we add the element to the array + # + # Handled cases: + # [Array, Hash] + elsif first_element.is_a?(Array) && second_element.is_a?(Hash) + merge_hash_into_array(first_element, second_element) + elsif first_element.is_a?(Hash) && second_element.is_a?(Array) + merge_hash_into_array(second_element, first_element) + # If both of them are hashes, we can deep_merge with the same logic + # + # Handled cases: + # [Hash, Hash] + elsif first_element.is_a?(Hash) && second_element.is_a?(Hash) + first_element.deep_merge(second_element) { |key, first, second| merge_hash_tree(first, second) } + # If both elements are arrays, we try to merge each element separatly + # + # Handled cases + # [Array, Array] + elsif first_element.is_a?(Array) && second_element.is_a?(Array) + first_element.map { |child_element| merge_hash_tree(child_element, second_element) } + # If one or both elements are a GroupDescendant, we wrap create an array + # combining them. + # + # Handled cases: + # [Object, Object], [Array, Array] + else + (Array.wrap(first_element) + Array.wrap(second_element)).uniq + end + end + + def crushable?(element) + element.is_a?(Hash) || element.is_a?(Array) + end + + def crush_hash(hash) + hash.flat_map do |key, value| + crushed_value = crushable?(value) ? crush(value) : value + Array.wrap(key) + Array.wrap(crushed_value) + end + end + + def crush_array(array) + array.flat_map do |element| + crushable?(element) ? crush(element) : element + end + end + end + end +end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb new file mode 100644 index 00000000000..d519d8e86fa --- /dev/null +++ b/lib/peek/views/gitaly.rb @@ -0,0 +1,34 @@ +module Peek + module Views + class Gitaly < View + def duration + ::Gitlab::GitalyClient.query_time + end + + def calls + ::Gitlab::GitalyClient.get_request_count + end + + def results + { duration: formatted_duration, calls: calls } + end + + private + + def formatted_duration + ms = duration * 1000 + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + + def setup_subscribers + subscribe 'start_processing.action_controller' do + ::Gitlab::GitalyClient.query_time = 0 + end + end + end + end +end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index b4d05f5995a..930b4bc13e2 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -5,7 +5,9 @@ namespace :gitlab do opts = if ENV['CI'] { - ce_repo: ENV['CI_REPOSITORY_URL'], + # We don't use CI_REPOSITORY_URL since it includes `gitlab-ci-token:xxxxxxxxxxxxxxxxxxxx@` + # which is confusing in the steps suggested in the job's output. + ce_repo: "#{ENV['CI_PROJECT_URL']}.git", branch: ENV['CI_COMMIT_REF_NAME'] } else |