diff options
Diffstat (limited to 'lib')
38 files changed, 629 insertions, 276 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 00d494f02f5..3fc2b453eb6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -53,7 +53,7 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events expose :enable_ssl_verification end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 6236fdd43ca..322624c6092 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -2,11 +2,11 @@ module API module Helpers module CommonHelpers def convert_parameters_from_legacy_format(params) - if params[:assignee_id].present? - params[:assignee_ids] = [params.delete(:assignee_id)] + params.tap do |params| + if params[:assignee_id].present? + params[:assignee_ids] = [params.delete(:assignee_id)] + end end - - params end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 40acaebf670..3d83720b7b9 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -56,16 +56,7 @@ module API authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) - users = User.all - users = User.where(username: params[:username]) if params[:username] - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - - if current_user.admin? - users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider] - users = users.external if params[:external] - end + users = UsersFinder.new(current_user, params).execute entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic present paginate(users), with: entity diff --git a/lib/github/import.rb b/lib/github/import.rb index 06beb607a3e..9c7eb965f93 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,4 +1,5 @@ require_relative 'error' + module Github class Import include Gitlab::ShellAdapter @@ -6,6 +7,7 @@ module Github class MergeRequest < ::MergeRequest self.table_name = 'merge_requests' + self.reset_callbacks :create self.reset_callbacks :save self.reset_callbacks :commit self.reset_callbacks :update @@ -16,6 +18,7 @@ module Github self.table_name = 'issues' self.reset_callbacks :save + self.reset_callbacks :create self.reset_callbacks :commit self.reset_callbacks :update self.reset_callbacks :validate @@ -79,7 +82,7 @@ module Github def fetch_repository begin project.create_repository unless project.repository.exists? - project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 9b9a0a8125a..a78a85397bd 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -21,7 +21,13 @@ module Gitlab def validate_variables(variables) variables.is_a?(Hash) && - variables.all? { |key, value| validate_string(key) && validate_string(value) } + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) end def validate_string(value) diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c3b0e651c3a..8acab605c91 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -15,6 +15,10 @@ module Gitlab def self.default {} end + + def value + Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + end end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 990b719ecfd..6e73361cad1 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -3,16 +3,33 @@ module Gitlab class FileCollection ConflictSideMissing = Class.new(StandardError) - attr_reader :merge_request, :our_commit, :their_commit + attr_reader :merge_request, :our_commit, :their_commit, :project - def initialize(merge_request) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit - end + delegate :repository, to: :project + + class << self + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def for_resolution(merge_request) + project = merge_request.source_project + + new(merge_request, project).tap do |file_collection| + project. + repository. + with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + + yield file_collection + end + end + end - def repository - merge_request.project.repository + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + def read_only(merge_request) + new(merge_request, merge_request.target_project) + end end def merge_index @@ -55,6 +72,15 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end + + private + + def initialize(merge_request, project) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + @project = project + end end end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 1ff34553f0a..e81d19a7a2e 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -11,6 +11,7 @@ module Gitlab # ref: String, # user_id: String, # user_name: String, + # user_username: String, # user_email: String # project_id: String, # repository: { @@ -51,6 +52,7 @@ module Gitlab message: message, user_id: user.id, user_name: user.name, + user_username: user.username, user_email: user.email, user_avatar: user.avatar_url, project_id: project.id, diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb new file mode 100644 index 00000000000..b42dc052949 --- /dev/null +++ b/lib/gitlab/data_builder/repository.rb @@ -0,0 +1,35 @@ +module Gitlab + module DataBuilder + module Repository + extend self + + # Produce a hash of post-receive data + def update(project, user, changes, refs) + { + event_name: 'repository_update', + + user_id: user.id, + user_name: user.name, + user_email: user.email, + user_avatar: user.avatar_url, + + project_id: project.id, + project: project.hook_attrs, + + changes: changes, + + refs: refs + } + end + + # Produce a hash of partial data for a single change + def single_change(oldrev, newrev, ref) + { + before: oldrev, + after: newrev, + ref: ref + } + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index f04a907004c..e76c9abbe04 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -283,11 +283,14 @@ module Gitlab add_column(table, new, new_type, limit: old_col.limit, - default: old_col.default, - null: old_col.null, precision: old_col.precision, scale: old_col.scale) + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new, old_col.default) if old_col.default + trigger_name = rename_trigger_name(table, old, new) quoted_table = quote_table_name(table) quoted_old = quote_column_name(old) @@ -303,6 +306,8 @@ module Gitlab update_column_in_batches(table, new, Arel::Table.new(table)[old]) + change_column_null(table, new, false) unless old_col.null + copy_indexes(table, old, new) copy_foreign_keys(table, old, new) end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index de4e6e7c404..5397877b5d5 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -15,7 +15,7 @@ module Gitlab end def path_patterns - @path_patterns ||= paths.map { |path| "%#{path}" } + @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] } end def rename_path_for_routable(routable) diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb new file mode 100644 index 00000000000..c45ae8feb2c --- /dev/null +++ b/lib/gitlab/dependency_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + LINKERS = [ + GemfileLinker + ].freeze + + def self.linker(blob_name) + LINKERS.find { |linker| linker.support?(blob_name) } + end + + def self.link(blob_name, plain_text, highlighted_text) + linker = linker(blob_name) + return highlighted_text unless linker + + linker.link(plain_text, highlighted_text) + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb new file mode 100644 index 00000000000..5f4027e7e81 --- /dev/null +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -0,0 +1,109 @@ +module Gitlab + module DependencyLinker + class BaseLinker + class_attribute :file_type + + def self.support?(blob_name) + Gitlab::FileDetector.type_of(blob_name) == file_type + end + + def self.link(*args) + new(*args).link + end + + attr_accessor :plain_text, :highlighted_text + + def initialize(plain_text, highlighted_text) + @plain_text = plain_text + @highlighted_text = highlighted_text + end + + def link + link_dependencies + + highlighted_lines.join.html_safe + end + + private + + def package_url(name) + raise NotImplementedError + end + + def link_dependencies + raise NotImplementedError + end + + def package_link(name, url = package_url(name)) + return name unless url + + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + end + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call("gem") + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call("gem", "specific_package") + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call("github", /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call("name") + # # Will link `package` in `self.name = "package"` + def link_method_call(method_names, value = nil, &url_proc) + value = + case value + when String + Regexp.escape(value) + when nil + /[^'"]+/ + else + value + end + + method_names = Array(method_names).map { |name| Regexp.escape(name) } + + regex = %r{ + #{Regexp.union(method_names)} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + + # Links package names based on regex. + # + # Example: + # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) + # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` + def link_regex(regex) + highlighted_lines.map!.with_index do |rich_line, i| + marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + + marker.mark(regex, group: :name) do |text, left:, right:| + url = block_given? ? yield(text) : package_url(text) + package_link(text, url) + end + end + end + + def plain_lines + @plain_lines ||= plain_text.lines + end + + def highlighted_lines + @highlighted_lines ||= highlighted_text.lines + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb new file mode 100644 index 00000000000..9b82e126528 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -0,0 +1,29 @@ +module Gitlab + module DependencyLinker + class GemfileLinker < BaseLinker + self.file_type = :gemfile + + private + + def link_dependencies + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call("gem") + + # Link `github: "user/repo"` to https://github.com/user/repo + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| + "https://github.com/#{name}" + end + + # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + + # Link `source "https://rubygems.org"` to https://rubygems.org + link_method_call("source", %r{https?://[^'"]+}) { |url| url } + end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb new file mode 100644 index 00000000000..c2a2eb15931 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -0,0 +1,17 @@ +module Gitlab + module Diff + class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + }.freeze + + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + symbol = MARKDOWN_SYMBOLS[mode] + "{#{symbol}#{text}#{symbol}}" + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 736933b1c4b..919965100ae 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,137 +1,21 @@ module Gitlab module Diff - class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - }.freeze - - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) - end - - def mark(line_inline_diffs, mode: nil, markdown: false) - return rich_line unless line_inline_diffs - - marker_ranges = [] - line_inline_diffs.each do |inline_diff_range| - # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range].flatten - # Turn the array of character positions into ranges - marker_ranges.concat(collapse_ranges(inline_diff_positions)) - end - - offset = 0 - - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "<span class='#{html_class_names(marker_ranges, mode, index)}'>" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "</span>" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + class InlineDiffMarker < Gitlab::StringRangeMarker + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - - rich_line.html_safe end private - def html_class_names(marker_ranges, mode, index) + def html_class_names(left, right, mode) class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 + class_names << "left" if left + class_names << "right" if right class_names << mode if mode class_names.join(" ") end - - # Mapping of character positions in the raw line, to the rich (highlighted) line - def position_mapping - @position_mapping ||= begin - mapping = [] - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - rich_char = rich_line[rich_pos] - - # The raw and rich lines are the same except for HTML tags, - # so skip over any `<...>` segment - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - # multi-char HTML entities in the rich line correspond to a single character in the raw line - if rich_char == '&' - multichar_mapping = [rich_pos] - until rich_char == ';' - rich_pos += 1 - multichar_mapping << rich_pos - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = multichar_mapping - else - mapping[raw_pos] = rich_pos - end - - rich_pos += 1 - end - - mapping - end - end - - # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - # Inserts tags around the characters identified by the given range - def insert_around_range(text, range, before, after, offset = 0) - # Just to be sure - return offset if offset + range.end + 1 > text.length - - text.insert(offset + range.begin, before) - offset += before.length - - text.insert(offset + range.end + 1, after) - offset += after.length - - offset - end end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 31a5b9d108b..ba31041d0c1 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -7,8 +7,8 @@ module Gitlab # - Don't contain a reserved word (expect for the words used in the # regex itself) # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route - # - Ending in `issues/id`/rendered_title` for the `issue_title` route - USED_IN_ROUTES = %w[noteable issue notes issues rendered_title + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes commit pipelines merge_requests new].freeze RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) @@ -18,7 +18,7 @@ module Gitlab 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/realtime_changes\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index f8b3d0b4965..c6a89597b23 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -12,6 +12,7 @@ module Gitlab version: 'version', gitignore: '.gitignore', koding: '.koding.yml', + gemfile: /\A(Gemfile|gems\.rb)\z/, gitlab_ci: '.gitlab-ci.yml', avatar: /\Alogo\.(png|jpg|gif)\z/, route_map: 'route-map.yml' diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb new file mode 100644 index 00000000000..093d9ed8092 --- /dev/null +++ b/lib/gitlab/file_finder.rb @@ -0,0 +1,32 @@ +# This class finds files in a repository by name and content +# the result is joined and sorted by file name +module Gitlab + class FileFinder + BATCH_SIZE = 100 + + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + end + + def find(query) + blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = Gitlab::ProjectSearchResults.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| + results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 019be151353..31d1b66b4f7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -183,6 +183,8 @@ module Gitlab when Gitaly::CommitDiffResponse init_from_gitaly(raw_diff) prune_diff_if_eligible(collapse) + when Gitaly::CommitDelta + init_from_gitaly(raw_diff) when nil raise "Nil as raw diff passed" else @@ -278,15 +280,15 @@ module Gitlab end end - def init_from_gitaly(diff_msg) - @diff = diff_msg.raw_chunks.join - @new_path = encode!(diff_msg.to_path.dup) - @old_path = encode!(diff_msg.from_path.dup) - @a_mode = diff_msg.old_mode.to_s(8) - @b_mode = diff_msg.new_mode.to_s(8) - @new_file = diff_msg.from_id == BLANK_SHA - @renamed_file = diff_msg.from_path != diff_msg.to_path - @deleted_file = diff_msg.to_id == BLANK_SHA + def init_from_gitaly(msg) + @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) + @new_path = encode!(msg.to_path.dup) + @old_path = encode!(msg.from_path.dup) + @a_mode = msg.old_mode.to_s(8) + @b_mode = msg.new_mode.to_s(8) + @new_file = msg.from_id == BLANK_SHA + @renamed_file = msg.from_path != msg.to_path + @deleted_file = msg.to_id == BLANK_SHA end def prune_diff_if_eligible(collapse = false) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 4e45ec7c174..bcbad8ec829 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -15,7 +15,6 @@ module Gitlab @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file @all_diffs = !!options.fetch(:all_diffs, false) @no_collapse = !!options.fetch(:no_collapse, true) - @deltas_only = !!options.fetch(:deltas_only, false) @line_count = 0 @byte_count = 0 @@ -27,8 +26,6 @@ module Gitlab if @populated # @iterator.each is slower than just iterating the array in place @array.each(&block) - elsif @deltas_only - each_delta(&block) else Gitlab::GitalyClient.migrate(:commit_raw_diffs) do each_patch(&block) @@ -81,14 +78,6 @@ module Gitlab files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes end - def each_delta - @iterator.each_delta.with_index do |delta, i| - diff = Gitlab::Git::Diff.new(delta) - - yield @array[i] = diff - end - end - def each_patch @iterator.each_with_index do |raw, i| # First yield cached Diff instances from @array diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 256318cb833..d380c5021ee 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -27,13 +27,15 @@ module Gitlab # Rugged repo object attr_reader :rugged + attr_reader :storage + # 'path' must be the path to a _bare_ git repository, e.g. # /path/to/my-repo.git - def initialize(repository_storage, relative_path) - @repository_storage = repository_storage + def initialize(storage, relative_path) + @storage = storage @relative_path = relative_path - storage_path = Gitlab.config.repositories.storages[@repository_storage]['path'] + storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) @@ -114,7 +116,7 @@ module Gitlab # Returns the number of valid branches def branch_count - Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + gitaly_migrate(:branch_names) do |is_enabled| if is_enabled gitaly_ref_client.count_branch_names else @@ -133,7 +135,7 @@ module Gitlab # Returns the number of valid tags def tag_count - Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + gitaly_migrate(:tag_names) do |is_enabled| if is_enabled gitaly_ref_client.count_tag_names else @@ -471,7 +473,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) # NOTE: This feature is intentionally disabled until # https://gitlab.com/gitlab-org/gitaly/issues/180 is resolved - # Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| + # gitaly_migrate(:find_ref_name) do |is_enabled| # if is_enabled # gitaly_ref_client.find_ref_name(sha, ref_path) # else @@ -965,11 +967,7 @@ module Gitlab end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@repository_storage, @relative_path) - end - - def gitaly_channel - Gitlab::GitalyClient.get_channel(@repository_storage) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end private diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 0e14253ab4e..742118b76a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -13,6 +13,16 @@ module Gitlab super(identifier, project, revision) end + def changes_refs + return enum_for(:changes_refs) unless block_given? + + changes.each do |change| + oldrev, newrev, ref = change.strip.split(' ') + + yield oldrev, newrev, ref + end + end + private def deserialize_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c69676a1dac..72466700c05 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -4,56 +4,42 @@ module Gitlab module GitalyClient SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze - # This function is not thread-safe because it updates Hashes in instance variables. - def self.configure_channels - @addresses = {} - @channels = {} - Gitlab.config.repositories.storages.each do |name, params| - address = params['gitaly_address'] - unless address.present? - raise "storage #{name.inspect} is missing a gitaly_address" - end + MUTEX = Mutex.new + private_constant :MUTEX - unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + def self.stub(name, storage) + MUTEX.synchronize do + @stubs ||= {} + @stubs[storage] ||= {} + @stubs[storage][name] ||= begin + klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub) + addr = address(storage) + addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' + klass.new(addr, :this_channel_is_insecure) end - - @addresses[name] = address - @channels[name] = new_channel(address) end end - def self.new_channel(address) - address = address.sub(%r{^tcp://}, '') if URI(address).scheme == 'tcp' - # NOTE: When Gitaly runs on a Unix socket, permissions are - # handled using the file system and no additional authentication is - # required (therefore the :this_channel_is_insecure flag) - # TODO: Add authentication support when Gitaly is running on a TCP socket. - GRPC::Core::Channel.new(address, {}, :this_channel_is_insecure) + def self.clear_stubs! + MUTEX.synchronize do + @stubs = nil + end end - def self.get_channel(storage) - if !Rails.env.production? && @channels.nil? - # In development mode the Rails auto-loader may reset the instance - # variables of this class. What we do here is not thread-safe. In normal - # circumstances (including production) these instance variables have - # been initialized from config/initializers. - configure_channels - end + def self.address(storage) + params = Gitlab.config.repositories.storages[storage] + raise "storage not found: #{storage.inspect}" if params.nil? - @channels[storage] - end + address = params['gitaly_address'] + unless address.present? + raise "storage #{storage.inspect} is missing a gitaly_address" + end - def self.get_address(storage) - if !Rails.env.production? && @addresses.nil? - # In development mode the Rails auto-loader may reset the instance - # variables of this class. What we do here is not thread-safe. In normal - # circumstances (including development) these instance variables have - # been initialized from config/initializers. - configure_channels + unless URI(address).scheme.in?(%w(tcp unix)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" end - @addresses[storage] + address end def self.enabled? diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 8e9323b05e1..4491903d788 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,41 +5,55 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - attr_accessor :stub - def initialize(repository) @gitaly_repo = repository.gitaly_repository - @stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) + @repository = repository end def is_ancestor(ancestor_id, child_id) + stub = GitalyClient.stub(:commit, @repository.storage) request = Gitaly::CommitIsAncestorRequest.new( repository: @gitaly_repo, ancestor_id: ancestor_id, child_id: child_id ) - @stub.commit_is_ancestor(request).value + stub.commit_is_ancestor(request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) + Gitlab::Git::DiffCollection.new(response, options) end - class << self - def diff_from_parent(commit, options = {}) - repository = commit.project.repository - gitaly_repo = repository.gitaly_repository - stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: repository.gitaly_channel) - parent = commit.parents[0] - parent_id = parent ? parent.id : EMPTY_TREE_ID - request = Gitaly::CommitDiffRequest.new( - repository: gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - ignore_whitespace_change: options.fetch(:ignore_whitespace_change, false), - paths: options.fetch(:paths, []) - ) - - Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + def commit_deltas(commit) + request_params = commit_diff_request_params(commit) + + response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def diff_service_stub + GitalyClient.stub(:diff, @repository.storage) + end end end end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb index a94a54883db..719554eac52 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -6,7 +6,7 @@ module Gitlab # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository - @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: repository.gitaly_channel) + @stub = GitalyClient.stub(:notifications, repository.storage) end def post_receive diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index f6c77ef1a3e..bf04e1fa50b 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -6,7 +6,7 @@ module Gitlab # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository - @stub = Gitaly::Ref::Stub.new(nil, nil, channel_override: repository.gitaly_channel) + @stub = GitalyClient.stub(:ref, repository.storage) end def default_branch_name diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d787d5db4a0..83bc230df3e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -13,6 +13,8 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end + attr_reader :blob_name + def initialize(blob_name, blob_content, repository: nil) @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @@ -21,16 +23,9 @@ module Gitlab end def highlight(text, continue: true, plain: false) - if plain - hl_lexer = Rouge::Lexers::PlainText - continue = false - else - hl_lexer = self.lexer - end - - @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe - rescue - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + highlighted_text = highlight_text(text, continue: continue, plain: plain) + highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text end def lexer @@ -50,5 +45,27 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end + + def highlight_text(text, continue: true, plain: false) + if plain + highlight_plain(text) + else + highlight_rich(text, continue: continue) + end + end + + def highlight_plain(text) + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + def highlight_rich(text, continue: true) + @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + rescue + highlight_plain(text) + end + + def link_dependencies(text, highlighted_text) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index c6dfa4ad9bd..cb8db2f1e9f 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -49,6 +49,9 @@ module Gitlab end end end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) end def self.prepare_metrics(metrics) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 47cfe412715..561aa9e162c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -84,23 +84,7 @@ module Gitlab def blobs return [] unless Ability.allowed?(@current_user, :download_code, @project) - @blobs ||= begin - blobs = project.repository.search_files_by_content(query, repository_ref).first(100) - found_file_names = Set.new - - results = blobs.map do |blob| - blob = self.class.parse_search_result(blob) - found_file_names << blob.filename - - [blob.filename, blob] - end - - project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| - results << [filename, nil] unless found_file_names.include?(filename) - end - - results.sort_by(&:first) - end + @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) end def wiki_blobs diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb new file mode 100644 index 00000000000..2a2eb4ae57f --- /dev/null +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -0,0 +1,26 @@ +module Gitlab + module Prometheus + module Queries + class BaseQuery + attr_accessor :client + delegate :query_range, :query, to: :client, prefix: true + + def raw_memory_usage_query(environment_slug) + %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} + end + + def raw_cpu_usage_query(environment_slug) + %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} + end + + def initialize(client) + @client = client + end + + def query(*args) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb new file mode 100644 index 00000000000..2cc08731f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -0,0 +1,26 @@ +module Gitlab::Prometheus::Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug + + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), + + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb new file mode 100644 index 00000000000..01d756d7284 --- /dev/null +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -0,0 +1,20 @@ +module Gitlab::Prometheus::Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus_client.rb index 37125980b1c..5b51a1779dd 100644 --- a/lib/gitlab/prometheus.rb +++ b/lib/gitlab/prometheus_client.rb @@ -2,7 +2,7 @@ module Gitlab PrometheusError = Class.new(StandardError) # Helper methods to interact with Prometheus network services & resources - class Prometheus + class PrometheusClient attr_reader :api_url def initialize(api_url:) @@ -15,7 +15,7 @@ module Gitlab def query(query, time: Time.now) get_result('vector') do - json_api_get('query', query: query, time: time.utc.to_f) + json_api_get('query', query: query, time: time.to_f) end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb new file mode 100644 index 00000000000..94fba0a221a --- /dev/null +++ b/lib/gitlab/string_range_marker.rb @@ -0,0 +1,102 @@ +module Gitlab + class StringRangeMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(marker_ranges) + return rich_line unless marker_ranges + + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + + offset = 0 + # Mark each range + rich_marker_ranges.each_with_index do |range, i| + offset_range = (range.begin + offset)..(range.end + offset) + original_text = rich_line[offset_range] + + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + + rich_line[offset_range] = text + + offset += text.length - original_text.length + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + end +end diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb new file mode 100644 index 00000000000..7ebf1c0428c --- /dev/null +++ b/lib/gitlab/string_regex_marker.rb @@ -0,0 +1,13 @@ +module Gitlab + class StringRegexMarker < StringRangeMarker + def mark(regex, group: 0, &block) + regex_match = raw_line.match(regex) + return rich_line unless regex_match + + begin_index, end_index = regex_match.offset(group) + name_range = begin_index..(end_index - 1) + + super([name_range], &block) + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 14d8e925d0e..4382cf7b12f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -52,6 +52,7 @@ module Gitlab def license_usage_data usage_data = { uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, recorded_at: Time.now, diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 351e2b10595..fe37e4da94f 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -26,7 +26,7 @@ module Gitlab } if Gitlab.config.gitaly.enabled - address = Gitlab::GitalyClient.get_address(project.repository_storage) + address = Gitlab::GitalyClient.address(project.repository_storage) params[:Repository] = repository.gitaly_repository.to_h feature_enabled = case action.to_s |