diff options
Diffstat (limited to 'lib/gitlab')
66 files changed, 1862 insertions, 579 deletions
diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb new file mode 100644 index 00000000000..b366c89889e --- /dev/null +++ b/lib/gitlab/akismet_helper.rb @@ -0,0 +1,39 @@ +module Gitlab + module AkismetHelper + def akismet_enabled? + current_application_settings.akismet_enabled + end + + def akismet_client + @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, + Gitlab.config.gitlab.url) + end + + def check_for_spam?(project, user) + akismet_enabled? && !project.team.member?(user) + end + + def is_spam?(environment, user, text) + client = akismet_client + ip_address = environment['REMOTE_ADDR'] + user_agent = environment['HTTP_USER_AGENT'] + + params = { + type: 'comment', + text: text, + created_at: DateTime.now, + author: user.name, + author_email: user.email, + referrer: environment['HTTP_REFERER'], + } + + begin + is_spam, is_blatant = client.check(ip_address, user_agent, params) + is_spam || is_blatant + rescue => e + Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") + false + end + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index b203b9d70e4..0b9c2e730f9 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -31,9 +31,7 @@ module Gitlab html = ::Asciidoctor.convert(input, asciidoc_opts) - if context[:project] - html = Banzai.render(html, context.merge(pipeline: :asciidoc)) - end + html = Banzai.post_process(html, context) html.html_safe end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 459e3d6bcdb..b9bb6e76081 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -36,7 +36,7 @@ module Gitlab # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '240']) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) raise Error, output unless status.zero? true end @@ -47,7 +47,7 @@ module Gitlab # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new.git") + # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") # def mv_repository(path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', @@ -150,6 +150,18 @@ module Gitlab "#{path}.git", tag_name]) end + # Gc repository + # + # path - project path with namespace + # + # Ex. + # gc("gitlab/gitlab-ci") + # + def gc(path) + Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', + "#{path}.git"]) + end + # Add new key to gitlab-shell # # Ex. diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 2355b3c6ddc..46e51a4bf6d 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -13,12 +13,36 @@ module Gitlab end def execute - project_identifier = project.import_source + import_issues if has_issues? - return true unless client.project(project_identifier)["has_issues"] + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error.new, e.message + ensure + Gitlab::BitbucketImport::KeyDeleter.new(project).execute + end - #Issues && Comments - issues = client.issues(project_identifier) + private + + def gl_user_id(project, bitbucket_id) + if bitbucket_id + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + else + project.creator_id + end + end + + def identifier + project.import_source + end + + def has_issues? + client.project(identifier)["has_issues"] + end + + def import_issues + issues = client.issues(identifier) issues.each do |issue| body = '' @@ -33,7 +57,7 @@ module Gitlab body = @formatter.author_line(author) body += issue["content"] - comments = client.issue_comments(project_identifier, issue["local_id"]) + comments = client.issue_comments(identifier, issue["local_id"]) if comments.any? body += @formatter.comments_header @@ -52,24 +76,13 @@ module Gitlab project.issues.create!( description: body, title: issue["title"], - state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', + state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', author_id: gl_user_id(project, reporter) ) end - - true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end - - private - - def gl_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end - end end end end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb new file mode 100644 index 00000000000..997a22779a0 --- /dev/null +++ b/lib/gitlab/blame.rb @@ -0,0 +1,55 @@ +module Gitlab + class Blame + attr_accessor :blob, :commit + + def initialize(blob, commit) + @blob = blob + @commit = commit + end + + def groups(highlight: true) + prev_sha = nil + groups = [] + current_group = nil + + i = 0 + blame.each do |commit, line| + commit = Commit.new(commit, project) + + sha = commit.sha + if prev_sha != sha + groups << current_group if current_group + current_group = { commit: commit, lines: [] } + end + + line = highlighted_lines[i].html_safe if highlight + current_group[:lines] << line + + prev_sha = sha + i += 1 + end + groups << current_group if current_group + + groups + end + + private + + def blame + @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path) + end + + def highlighted_lines + @blob.load_all_data!(repository) + @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + end + + def project + commit.project + end + + def repository + project.repository + end + end +end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/build_data_builder.rb index 86bfa0a4378..34e949130da 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/build_data_builder.rb @@ -23,6 +23,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_allow_failure: build.allow_failure, # TODO: do we still need it? project_id: project.id, diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..f2020c82d40 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,111 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + class ParserError < StandardError; end + + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + + attr_reader :file, :path, :full_version + + def initialize(file, path, **opts) + @file, @path, @opts = file, path, opts + @full_version = read_version + end + + def version + @full_version.match(VERSION_PATTERN)[1] + end + + def errors + gzip do |gz| + read_string(gz) # version + errors = read_string(gz) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end + end + end + + def find_entries! + gzip do |gz| + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) + end + end + + def to_entry + entries = find_entries! + Entry.new(@path, entries) + end + + private + + def match_entries(gz) + entries = {} + + child_pattern = '[^/]*/?$' unless @opts[:recursive] + match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ + + until gz.eof? do + begin + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + + next unless path.valid_encoding? && meta.valid_encoding? + next unless path =~ match_pattern + next if path =~ INVALID_PATH_PATTERN + + entries[path] = JSON.parse(meta, symbolize_names: true) + rescue JSON::ParserError, Encoding::CompatibilityError + next + end + end + + entries + end + + def read_version + gzip do |gz| + version_string = read_string(gz) + + unless version_string + raise ParserError, 'Artifacts metadata file empty!' + end + + unless version_string =~ VERSION_PATTERN + raise ParserError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_uint32(gz) + return nil unless string_size + gz.read(string_size) + end + + def gzip(&block) + Zlib::GzipReader.open(@file, &block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..7f4c750b6fd --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,126 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entries + attr_accessor :name + + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') + @entries = entries + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path) + end + + def empty? + children.empty? + end + + def total_size + descendant_pattern = %r{^#{Regexp.escape(@path)}} + entries.sum do |path, entry| + (entry[:size] if path =~ descendant_pattern).to_i + end + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end + end + end + end +end diff --git a/lib/gitlab/compare_result.rb b/lib/gitlab/compare_result.rb deleted file mode 100644 index 0d696a1ee28..00000000000 --- a/lib/gitlab/compare_result.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - class CompareResult - attr_reader :commits, :diffs - - def initialize(compare, diff_options = {}) - @commits, @diffs = compare.commits, compare.diffs(nil, diff_options) - end - end -end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 8a7f8dc5003..85583dce9ee 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -45,11 +45,11 @@ module Gitlab end def starting_year - (Time.now - 1.year).strftime("%Y") + 1.year.ago.year end def starting_month - Date.today.strftime("%m").to_i + Date.today.month end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7a86c09158e..761b63e98f6 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -4,11 +4,14 @@ module Gitlab key = :current_application_settings RequestStore.store[key] ||= begin + settings = nil + if connect_to_db? - ApplicationSetting.current || ApplicationSetting.create_from_defaults - else - fake_application_settings + settings = ::ApplicationSetting.current + settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end + + settings || fake_application_settings end end @@ -18,29 +21,36 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], + twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], sign_in_text: Settings.extra['sign_in_text'], restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], max_attachment_size: Settings.gitlab['max_attachment_size'], session_expire_delay: Settings.gitlab['session_expire_delay'], - import_sources: Settings.gitlab['import_sources'], + default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], + default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], + import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], + require_two_factor_authentication: false, + two_factor_grace_period: 48, + akismet_enabled: false ) end private def connect_to_db? - use_db = if ENV['USE_DB'] == "false" - false - else - true - end - - use_db && ActiveRecord::Base.connection.active? && - !ActiveRecord::Migrator.needs_migration? && - ActiveRecord::Base.connection.table_exists?('application_settings') + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false + + ENV['USE_DB'] != 'false' && + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + + rescue ActiveRecord::NoDatabaseError + false end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index de77a6fbff1..6f9da69983a 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,16 +1,23 @@ module Gitlab module Database + def self.adapter_name + connection.adapter_name + end + def self.mysql? - ActiveRecord::Base.connection.adapter_name.downcase == 'mysql2' + adapter_name.downcase == 'mysql2' end def self.postgresql? - ActiveRecord::Base.connection.adapter_name.downcase == 'postgresql' + adapter_name.downcase == 'postgresql' + end + + def self.version + database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end def true_value - case ActiveRecord::Base.connection.adapter_name.downcase - when 'postgresql' + if Gitlab::Database.postgresql? "'t'" else 1 @@ -18,12 +25,27 @@ module Gitlab end def false_value - case ActiveRecord::Base.connection.adapter_name.downcase - when 'postgresql' + if Gitlab::Database.postgresql? "'f'" else 0 end end + + private + + def self.connection + ActiveRecord::Base.connection + end + + def self.database_version + row = connection.execute("SELECT VERSION()").first + + if postgresql? + row['version'] + else + row.first + end + end end end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb new file mode 100644 index 00000000000..a78fde9d782 --- /dev/null +++ b/lib/gitlab/devise_failure.rb @@ -0,0 +1,23 @@ +module Gitlab + class DeviseFailure < Devise::FailureApp + protected + + # Override `Devise::FailureApp#request_format` to handle a special case + # + # This tells Devise to handle an unauthenticated `.zip` request as an HTML + # request (i.e., redirect to sign in). + # + # Otherwise, Devise would respond with a 401 Unauthorized with + # `Content-Type: application/zip` and a response body in plaintext, and the + # browser would freak out. + # + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 + def request_format + if request.format == :zip + Mime::Type.lookup_by_extension(:html).ref + else + super + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 79061cd0141..d2e85cabf72 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,18 +1,39 @@ module Gitlab module Diff class File - attr_reader :diff + attr_reader :diff, :diff_refs delegate :new_file, :deleted_file, :renamed_file, :old_path, :new_path, to: :diff, prefix: false - def initialize(diff) + def initialize(diff, diff_refs) @diff = diff + @diff_refs = diff_refs + end + + def old_ref + diff_refs[0] if diff_refs + end + + def new_ref + diff_refs[1] if diff_refs end # Array of Gitlab::DIff::Line objects def diff_lines - @lines ||= parser.parse(raw_diff.lines) + @lines ||= parser.parse(raw_diff.each_line).to_a + end + + def too_large? + diff.too_large? + end + + def highlighted_diff_lines + Gitlab::Diff::Highlight.new(self).highlight + end + + def parallel_diff_lines + Gitlab::Diff::ParallelDiff.new(self).parallelize end def mode_changed? diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb new file mode 100644 index 00000000000..9429b3ff88d --- /dev/null +++ b/lib/gitlab/diff/highlight.rb @@ -0,0 +1,77 @@ +module Gitlab + module Diff + class Highlight + attr_reader :diff_file, :diff_lines, :raw_lines + + delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + + def initialize(diff_lines) + if diff_lines.is_a?(Gitlab::Diff::File) + @diff_file = diff_lines + @diff_lines = @diff_file.diff_lines + else + @diff_lines = diff_lines + end + @raw_lines = @diff_lines.map(&:text) + end + + def highlight + @diff_lines.map.with_index do |diff_line, i| + diff_line = diff_line.dup + # ignore highlighting for "match" lines + next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + + rich_line = highlight_line(diff_line) || diff_line.text + + if line_inline_diffs = inline_diffs[i] + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) + end + + diff_line.text = rich_line + + diff_line + end + end + + private + + def highlight_line(diff_line) + return unless diff_file && diff_file.diff_refs + + line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + + case diff_line.type + when 'new', nil + rich_line = new_lines[diff_line.new_pos - 1] + when 'old' + rich_line = old_lines[diff_line.old_pos - 1] + end + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + "#{line_prefix}#{rich_line}".html_safe if rich_line + end + + def inline_diffs + @inline_diffs ||= InlineDiff.for_lines(@raw_lines) + end + + def old_lines + return unless diff_file + @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + end + + def new_lines + return unless diff_file + @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) + end + + def processing_args(version) + ref = send("diff_#{version}_ref") + path = send("diff_#{version}_path") + + [ref.project.repository, ref.id, path] + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb new file mode 100644 index 00000000000..789c14518b0 --- /dev/null +++ b/lib/gitlab/diff/inline_diff.rb @@ -0,0 +1,88 @@ +module Gitlab + module Diff + class InlineDiff + attr_accessor :old_line, :new_line, :offset + + def self.for_lines(lines) + local_edit_indexes = self.find_local_edits(lines) + + inline_diffs = [] + + local_edit_indexes.each do |index| + old_index = index + new_index = index + 1 + old_line = lines[old_index] + new_line = lines[new_index] + + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + + inline_diffs[old_index] = old_diffs + inline_diffs[new_index] = new_diffs + end + + inline_diffs + end + + def initialize(old_line, new_line, offset: 0) + @old_line = old_line[offset..-1] + @new_line = new_line[offset..-1] + @offset = offset + end + + def inline_diffs + # Skip inline diff if empty line was replaced with content + return if old_line == "" + + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + + private + + def self.find_local_edits(lines) + line_prefixes = lines.map { |line| line.match(/\A([+-])/) ? $1 : ' ' } + joined_line_prefixes = " #{line_prefixes.join} " + + offset = 0 + local_edit_indexes = [] + while index = joined_line_prefixes.index(" -+ ", offset) + local_edit_indexes << index + offset = index + 1 + end + + local_edit_indexes + end + + def longest_common_prefix(a, b) + max_length = [a.length, b.length].max + + length = 0 + (0..max_length - 1).each do |pos| + old_char = a[pos] + new_char = b[pos] + + break if old_char != new_char + length += 1 + end + + length + end + + def longest_common_suffix(a, b) + longest_common_prefix(a.reverse, b.reverse) + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb new file mode 100644 index 00000000000..dccb717e95d --- /dev/null +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -0,0 +1,115 @@ +module Gitlab + module Diff + class InlineDiffMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(line_inline_diffs) + return rich_line unless line_inline_diffs + + marker_ranges = [] + line_inline_diffs.each do |inline_diff_range| + # Map the inline-diff range based on the raw line to character positions in the rich line + inline_diff_positions = position_mapping[inline_diff_range].flatten + # Turn the array of character positions into ranges + marker_ranges.concat(collapse_ranges(inline_diff_positions)) + end + + offset = 0 + # Mark each range + marker_ranges.each_with_index do |range, i| + class_names = ["idiff"] + class_names << "left" if i == 0 + class_names << "right" if i == marker_ranges.length - 1 + + offset = insert_around_range(rich_line, range, "<span class='#{class_names.join(" ")}'>", "</span>", offset) + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + + # Inserts tags around the characters identified by the given range + def insert_around_range(text, range, before, after, offset = 0) + # Just to be sure + return offset if offset + range.end + 1 > text.length + + text.insert(offset + range.begin, before) + offset += before.length + + text.insert(offset + range.end + 1, after) + offset += after.length + + offset + end + end + end +end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0072194606e..03730b435ad 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,7 +1,8 @@ module Gitlab module Diff class Line - attr_reader :type, :text, :index, :old_pos, :new_pos + attr_reader :type, :index, :old_pos, :new_pos + attr_accessor :text def initialize(text, type, index, old_pos, new_pos) @text, @type, @index = text, type, index diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb new file mode 100644 index 00000000000..74f9b3c050a --- /dev/null +++ b/lib/gitlab/diff/parallel_diff.rb @@ -0,0 +1,119 @@ +module Gitlab + module Diff + class ParallelDiff + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + def parallelize + lines = [] + skip_next = false + + highlighted_diff_lines = diff_file.highlighted_diff_lines + highlighted_diff_lines.each do |line| + full_line = line.text + type = line.type + line_code = generate_line_code(diff_file.file_path, line) + line_new = line.new_pos + line_old = line.old_pos + + next_line = diff_file.next_line(line.index) + + if next_line + next_line = highlighted_diff_lines[next_line.index] + next_line_code = generate_line_code(diff_file.file_path, next_line) + next_type = next_line.type + next_line = next_line.text + end + + case type + when 'match', nil + # line in the right panel is the same as in the left one + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + when 'old' + case next_type + when 'new' + # Left side has text removed, right side has text added + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: line_new, + text: next_line, + line_code: next_line_code + } + } + skip_next = true + when 'old', 'nonewline', nil + # Left side has text removed, right side doesn't have any change + # No next line code, no new line number, no new line text + lines << { + left: { + type: type, + number: line_old, + text: full_line, + line_code: line_code, + }, + right: { + type: next_type, + number: nil, + text: "", + line_code: nil + } + } + end + when 'new' + if skip_next + # Change has been already included in previous line so no need to do it again + skip_next = false + next + else + # Change is only on the right side, left side has no change + lines << { + left: { + type: nil, + number: nil, + text: "", + line_code: line_code, + }, + right: { + type: type, + number: line_new, + text: full_line, + line_code: line_code + } + } + end + end + end + lines + end + + private + + def generate_line_code(file_path, line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..d0815fc7eea 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -4,49 +4,56 @@ module Gitlab include Enumerable def parse(lines) + return [] if lines.blank? + @lines = lines - lines_obj = [] line_obj_index = 0 line_old = 1 line_new = 1 type = nil - lines_arr = ::Gitlab::InlineDiff.processing lines - - lines_arr.each do |line| - next if filename?(line) - - full_line = html_escape(line.gsub(/\n/, '')) - full_line = ::Gitlab::InlineDiff.replace_markers full_line - - if line.match(/^@@ -/) - type = "match" - - line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 - line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 - - next if line_old <= 1 && line_new <= 1 #top of file - lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) - line_obj_index += 1 - next - else - type = identification_type(line) - lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) - line_obj_index += 1 - end - - - if line[0] == "+" - line_new += 1 - elsif line[0] == "-" - line_old += 1 - else - line_new += 1 - line_old += 1 + # By returning an Enumerator we make it possible to search for a single line (with #find) + # without having to instantiate all the others that come after it. + Enumerator.new do |yielder| + @lines.each do |line| + next if filename?(line) + + full_line = line.gsub(/\n/, '') + + if line.match(/^@@ -/) + type = "match" + + line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 + line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 + + next if line_old <= 1 && line_new <= 1 #top of file + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + next + elsif line[0] == '\\' + type = 'nonewline' + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + else + type = identification_type(line) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + line_obj_index += 1 + end + + + case line[0] + when "+" + line_new += 1 + when "-" + line_old += 1 + when "\\" + # No increment + else + line_new += 1 + line_old += 1 + end end end - - lines_obj end def empty? @@ -56,24 +63,21 @@ module Gitlab private def filename?(line) - line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', - '--- /tmp/diffy', '+++ /tmp/diffy') + line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit + '--- /tmp/diffy', '+++ /tmp/diffy') end def identification_type(line) - if line[0] == "+" + case line[0] + when "+" "new" - elsif line[0] == "-" + when "-" "old" else nil end end - - def html_escape(str) - replacements = { '&' => '&', '>' => '>', '<' => '<', '"' => '"', "'" => ''' } - str.gsub(/[&"'><]/, replacements) - end end end end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index a2eb7a70bd2..41f0edcaf7e 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -9,6 +9,7 @@ module Gitlab delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author + delegate :username, to: :author, prefix: :author def initialize(notify, project_id, recipient, opts = {}) raise ArgumentError, 'Missing options: author_id, ref, action' unless @@ -49,7 +50,7 @@ module Gitlab end def compare_timeout - compare.timeout if compare + diffs.overflow? if diffs end def reverse_compare? diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 2b252b32887..2ca21af5bc8 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -74,7 +74,7 @@ module Gitlab def sent_notification return nil unless reply_key - + SentNotification.for(reply_key) end @@ -82,10 +82,7 @@ module Gitlab attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) attachments.each do |link| - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - reply << "\n\n#{text}" + reply << "\n\n#{link[:markdown]}" end reply diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb new file mode 100644 index 00000000000..2ef50286b1d --- /dev/null +++ b/lib/gitlab/exclusive_lease.rb @@ -0,0 +1,41 @@ +module Gitlab + # This class implements an 'exclusive lease'. We call it a 'lease' + # because it has a set expiry time. We call it 'exclusive' because only + # one caller may obtain a lease for a given key at a time. The + # implementation is intended to work across GitLab processes and across + # servers. It is a 'cheap' alternative to using SQL queries and updates: + # you do not need to change the SQL schema to start using + # ExclusiveLease. + # + # It is important to choose the timeout wisely. If the timeout is very + # high (1 hour) then the throughput of your operation gets very low (at + # most once an hour). If the timeout is lower than how long your + # operation may take then you cannot count on exclusivity. For example, + # if the timeout is 10 seconds and you do an operation which may take 20 + # seconds then two overlapping operations may hold a lease for the same + # key at the same time. + # + class ExclusiveLease + def initialize(key, timeout:) + @key, @timeout = key, timeout + end + + # Try to obtain the lease. Return true on success, + # false if the lease is already taken. + def try_obtain + # Performing a single SET is atomic + !!redis.set(redis_key, '1', nx: true, ex: @timeout) + end + + private + + def redis + # Maybe someday we want to use a connection pool... + @redis ||= Redis.new(url: Gitlab::RedisConfig.url) + end + + def redis_key + "gitlab:exclusive_lease:#{@key}" + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 403ebeec474..db580b5e578 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -232,9 +232,7 @@ module Gitlab return nil if res.nil? - text = "[#{res['alt']}](#{res['url']})" - text = "!#{text}" if res['is_image'] - text + res[:markdown] end def build_attachment_url(rel_url) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index f065cc5e9e9..191bea86ac3 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,8 +1,8 @@ module Gitlab module Git - BLANK_SHA = '0' * 40 - TAG_REF_PREFIX = "refs/tags/" - BRANCH_REF_PREFIX = "refs/heads/" + BLANK_SHA = ('0' * 40).freeze + TAG_REF_PREFIX = "refs/tags/".freeze + BRANCH_REF_PREFIX = "refs/heads/".freeze class << self def ref_name(ref) diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb new file mode 100644 index 00000000000..a088e19d1e7 --- /dev/null +++ b/lib/gitlab/git_post_receive.rb @@ -0,0 +1,60 @@ +module Gitlab + class GitPostReceive + include Gitlab::Identifier + attr_reader :repo_path, :identifier, :changes, :project + + def initialize(repo_path, identifier, changes) + repo_path.gsub!(/\.git\z/, '') + repo_path.gsub!(/\A\//, '') + + @repo_path = repo_path + @identifier = identifier + @changes = deserialize_changes(changes) + + retrieve_project_and_type + end + + def wiki? + @type == :wiki + end + + def regular_project? + @type == :project + end + + def identify(revision) + super(identifier, project, revision) + end + + private + + def retrieve_project_and_type + @type = :project + @project = Project.find_with_namespace(@repo_path) + + if @repo_path.end_with?('.wiki') && !@project + @type = :wiki + @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, '')) + end + end + + def deserialize_changes(changes) + changes = Base64.decode64(changes) unless changes.include?(' ') + changes = utf8_encode_changes(changes) + changes.lines + end + + def utf8_encode_changes(changes) + changes = changes.dup + + changes.force_encoding('UTF-8') + return changes if changes.valid_encoding? + + # Convert non-UTF-8 branch/tag names to UTF-8 so they can be dumped as JSON. + detection = CharlockHolmes::EncodingDetector.detect(changes) + return changes unless detection && detection[:encoding] + + CharlockHolmes::Converter.convert(changes, detection[:encoding], 'UTF-8') + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb new file mode 100644 index 00000000000..202263c6742 --- /dev/null +++ b/lib/gitlab/github_import/base_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module GithubImport + class BaseFormatter + attr_reader :formatter, :project, :raw_data + + def initialize(project, raw_data) + @project = project + @raw_data = raw_data + @formatter = Gitlab::ImportFormatter.new + end + + private + + def gl_user_id(github_id) + User.joins(:identities). + find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). + try(:id) + end + end + end +end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb new file mode 100644 index 00000000000..7d58e53991a --- /dev/null +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class CommentFormatter < BaseFormatter + def attributes + { + project: project, + note: note, + commit_id: raw_data.commit_id, + line_code: line_code, + author_id: author_id, + created_at: raw_data.created_at, + updated_at: raw_data.updated_at + } + end + + private + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def line_code + if on_diff? + Gitlab::Diff::LineCode.generate(raw_data.path, raw_data.position, 0) + end + end + + def on_diff? + raw_data.path && raw_data.position + end + + def note + formatter.author_line(author) + body + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index b5720f6e2cb..172c5441e36 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -12,39 +14,86 @@ module Gitlab end def execute - #Issues && Comments + import_issues && import_pull_requests && import_wiki + end + + private + + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, - direction: :asc).each do |issue| - if issue.pull_request.nil? - - body = @formatter.author_line(issue.user.login) - body += issue.body || "" + direction: :asc).each do |raw_data| + gh_issue = IssueFormatter.new(project, raw_data) - if issue.comments > 0 - body += @formatter.comments_header + if gh_issue.valid? + issue = Issue.create!(gh_issue.attributes) - client.issue_comments(project.import_source, issue.number).each do |c| - body += @formatter.comment(c.user.login, c.created_at, c.body) - end + if gh_issue.has_comments? + import_comments(gh_issue.number, issue) end + end + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + + def import_pull_requests + client.pull_requests(project.import_source, state: :all, + sort: :created, + direction: :asc).each do |raw_data| + pull_request = PullRequestFormatter.new(project, raw_data) + + if pull_request.valid? + merge_request = MergeRequest.new(pull_request.attributes) - project.issues.create!( - description: body, - title: issue.title, - state: issue.state == 'closed' ? 'closed' : 'opened', - author_id: gl_user_id(project, issue.user.id) - ) + if merge_request.save + import_comments(pull_request.number, merge_request) + import_comments_on_diff(pull_request.number, merge_request) + end end end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message end - private + def import_comments(issue_number, noteable) + comments = client.issue_comments(project.import_source, issue_number) + create_comments(comments, noteable) + end + + def import_comments_on_diff(pull_request_number, merge_request) + comments = client.pull_request_comments(project.import_source, pull_request_number) + create_comments(comments, merge_request) + end - def gl_user_id(project, github_id) - user = User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s) - (user && user.id) || project.creator_id + def create_comments(comments, noteable) + comments.each do |raw_data| + comment = CommentFormatter.new(project, raw_data) + noteable.notes.create!(comment.attributes) + end + end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + + true + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + raise Projects::ImportService::Error, e.message + else + true + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb new file mode 100644 index 00000000000..1e3ba44f27c --- /dev/null +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssueFormatter < BaseFormatter + def attributes + { + project: project, + title: raw_data.title, + description: description, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def has_comments? + raw_data.comments > 0 + end + + def number + raw_data.number + end + + def valid? + raw_data.pull_request.nil? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + @formatter.author_line(author) + body + end + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb new file mode 100644 index 00000000000..4e507b090e8 --- /dev/null +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -0,0 +1,105 @@ +module Gitlab + module GithubImport + class PullRequestFormatter < BaseFormatter + def attributes + { + title: raw_data.title, + description: description, + source_project: source_project, + source_branch: source_branch.name, + target_project: target_project, + target_branch: target_branch.name, + state: state, + author_id: author_id, + assignee_id: assignee_id, + created_at: raw_data.created_at, + updated_at: updated_at + } + end + + def number + raw_data.number + end + + def valid? + !cross_project? && source_branch.present? && target_branch.present? + end + + private + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gl_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gl_user_id(raw_data.user.id) || project.creator_id + end + + def body + raw_data.body || "" + end + + def cross_project? + source_repo.present? && target_repo.present? && source_repo.id != target_repo.id + end + + def description + formatter.author_line(author) + body + end + + def source_project + project + end + + def source_repo + raw_data.head.repo + end + + def source_branch + source_project.repository.find_branch(raw_data.head.ref) + end + + def target_project + project + end + + def target_repo + raw_data.base.repo + end + + def target_branch + target_project.repository.find_branch(raw_data.base.ref) + end + + def state + @state ||= case true + when raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' + when raw_data.state == 'closed' + 'closed' + else + 'opened' + end + end + + def updated_at + case state + when 'merged' then raw_data.merged_at + when 'closed' then raw_data.closed_at + else + raw_data.updated_at + end + end + end + end +end diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6c592ff469c --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(/\.git\z/, ".wiki.git") + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e24b94d6159..850b73244c6 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -12,7 +12,7 @@ module Gitlab end def execute - project_identifier = URI.encode(project.import_source, '/') + project_identifier = CGI.escape(project.import_source) #Issues && Comments issues = client.issues(project_identifier) diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb new file mode 100644 index 00000000000..cac76442321 --- /dev/null +++ b/lib/gitlab/highlight.rb @@ -0,0 +1,39 @@ +module Gitlab + class Highlight + def self.highlight(blob_name, blob_content, nowrap: true) + new(blob_name, blob_content, nowrap: nowrap).highlight(blob_content, continue: false) + end + + def self.highlight_lines(repository, ref, file_name) + blob = repository.blob_at(ref, file_name) + return [] unless blob + + blob.load_all_data!(repository) + highlight(file_name, blob.data).lines.map!(&:html_safe) + end + + def initialize(blob_name, blob_content, nowrap: true) + @formatter = rouge_formatter(nowrap: nowrap) + @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + end + + def highlight(text, continue: true) + @formatter.format(@lexer.lex(text, continue: continue)).html_safe + rescue + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + private + + def rouge_formatter(options = {}) + options = options.reverse_merge( + nowrap: true, + cssclass: 'code highlight', + lineanchors: true, + lineanchorsid: 'LC' + ) + + Rouge::Formatters::HTMLGitlab.new(options) + end + end +end diff --git a/lib/gitlab/inline_diff.rb b/lib/gitlab/inline_diff.rb deleted file mode 100644 index 44507bde25d..00000000000 --- a/lib/gitlab/inline_diff.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Gitlab - class InlineDiff - class << self - - START = "#!idiff-start!#" - FINISH = "#!idiff-finish!#" - - def processing(diff_arr) - indexes = _indexes_of_changed_lines diff_arr - - indexes.each do |index| - first_line = diff_arr[index+1] - second_line = diff_arr[index+2] - - # Skip inline diff if empty line was replaced with content - next if first_line == "-\n" - - first_token = find_first_token(first_line, second_line) - apply_first_token(diff_arr, index, first_token) - - last_token = find_last_token(first_line, second_line, first_token) - apply_last_token(diff_arr, index, last_token) - end - - diff_arr - end - - def apply_first_token(diff_arr, index, first_token) - start = first_token + START - - if first_token.empty? - # In case if we remove string of spaces in commit - diff_arr[index+1].sub!("-", "-" => "-#{START}") - diff_arr[index+2].sub!("+", "+" => "+#{START}") - else - diff_arr[index+1].sub!(first_token, first_token => start) - diff_arr[index+2].sub!(first_token, first_token => start) - end - end - - def apply_last_token(diff_arr, index, last_token) - # This is tricky: escape backslashes so that `sub` doesn't interpret them - # as backreferences. Regexp.escape does NOT do the right thing. - replace_token = FINISH + last_token.gsub(/\\/, '\&\&') - diff_arr[index+1].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - diff_arr[index+2].sub!(/#{Regexp.escape(last_token)}$/, replace_token) - end - - def find_first_token(first_line, second_line) - max_length = [first_line.size, second_line.size].max - first_the_same_symbols = 0 - - (0..max_length + 1).each do |i| - first_the_same_symbols = i - 1 - - if first_line[i] != second_line[i] && i > 0 - break - end - end - - first_line[0..first_the_same_symbols][1..-1] - end - - def find_last_token(first_line, second_line, first_token) - max_length = [first_line.size, second_line.size].max - last_the_same_symbols = 0 - - (1..max_length + 1).each do |i| - last_the_same_symbols = -i - shortest_line = second_line.size > first_line.size ? first_line : second_line - - if (first_line[-i] != second_line[-i]) || "#{first_token}#{START}".size == shortest_line[1..-i].size - break - end - end - - last_the_same_symbols += 1 - first_line[last_the_same_symbols..-1] - end - - def _indexes_of_changed_lines(diff_arr) - chain_of_first_symbols = "" - diff_arr.each_with_index do |line, i| - chain_of_first_symbols += line[0] - end - chain_of_first_symbols.gsub!(/[^\-\+]/, "#") - - offset = 0 - indexes = [] - while index = chain_of_first_symbols.index("#-+#", offset) - indexes << index - offset = index + 1 - end - indexes - end - - def replace_markers(line) - line.gsub!(START, "<span class='idiff'>") - line.gsub!(FINISH, "</span>") - line - end - end - end -end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index c438a3d167b..da4435c7308 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -5,7 +5,7 @@ module Gitlab module LDAP class Access - attr_reader :adapter, :provider, :user + attr_reader :provider, :user def self.open(user, &block) Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter| @@ -32,20 +32,20 @@ module Gitlab end def allowed? - if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + if ldap_user return true unless ldap_config.active_directory # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.block + user.ldap_block false else - user.activate if user.blocked? && !ldap_config.block_auto_created_users + user.activate if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.block + user.ldap_block false end rescue @@ -59,6 +59,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(provider) end + + def ldap_user + @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 577a890a7d9..df65179bfea 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -70,19 +70,25 @@ module Gitlab end def ldap_search(*args) - results = ldap.search(*args) + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. + Timeout.timeout(config.timeout) do + results = ldap.search(*args) - if results.nil? - response = ldap.get_operation_result + if results.nil? + response = ldap.get_operation_result - unless response.code.zero? - Rails.logger.warn("LDAP search error: #{response.message}") - end + unless response.code.zero? + Rails.logger.warn("LDAP search error: #{response.message}") + end - [] - else - results + [] + else + results + end end + rescue Timeout::Error + Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") + [] end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 101a3285f4b..aff7ccb157f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -88,6 +88,10 @@ module Gitlab options['attributes'] end + def timeout + options['timeout'].to_i + end + protected def base_config Gitlab.config.ldap diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index aef08c97d1d..b84c81f1a6c 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -24,34 +24,41 @@ module Gitlab update_user_attributes end + def save + super('LDAP') + end + # instance methods def gl_user @gl_user ||= find_by_uid_and_provider || find_by_email || build_new_user end def find_by_uid_and_provider - self.class.find_by_uid_and_provider( - auth_hash.uid, auth_hash.provider) + self.class.find_by_uid_and_provider(auth_hash.uid, auth_hash.provider) end def find_by_email - ::User.find_by(email: auth_hash.email.downcase) + ::User.find_by(email: auth_hash.email.downcase) if auth_hash.has_email? end def update_user_attributes - return unless persisted? + if persisted? + if auth_hash.has_email? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. - identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } - identity ||= gl_user.identities.build(provider: auth_hash.provider) + # For a new identity set extern_uid to the LDAP DN + # For an existing identity with matching email but changed DN, update the DN. + # For an existing identity with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid + end - # For a new user set extern_uid to the LDAP DN - # For an existing user with matching email but changed DN, update the DN. - # For an existing user with no change in DN, this line changes nothing. - identity.extern_uid = auth_hash.uid + gl_user.ldap_email = auth_hash.has_email? gl_user end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 8f3f43c0e91..699d8b9fc07 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab module Markdown class Pipeline diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 2d266ccfe9e..88a265c6af2 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -6,16 +6,20 @@ module Gitlab METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s PATH_REGEX = /^#{RAILS_ROOT}\/?/ - def self.pool_size - current_application_settings[:metrics_pool_size] || 16 - end - - def self.timeout - current_application_settings[:metrics_timeout] || 10 + def self.settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15 + } end def self.enabled? - current_application_settings[:metrics_enabled] || false + settings[:enabled] || false end def self.mri? @@ -26,32 +30,13 @@ module Gitlab # This is memoized since this method is called for every instrumented # method. Loading data from an external cache on every method call slows # things down too much. - @method_call_threshold ||= - (current_application_settings[:metrics_method_call_threshold] || 10) + @method_call_threshold ||= settings[:method_call_threshold] end def self.pool @pool end - def self.hostname - @hostname - end - - # Returns a relative path and line number based on the last application call - # frame. - def self.last_relative_application_frame - frame = caller_locations.find do |l| - l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT) - end - - if frame - return frame.path.sub(PATH_REGEX, ''), frame.lineno - else - return nil, nil - end - end - def self.submit_metrics(metrics) prepared = prepare_metrics(metrics) @@ -85,19 +70,15 @@ module Gitlab value.to_s.gsub('=', '\\=') end - @hostname = Socket.gethostname - # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? - @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do - host = current_application_settings[:metrics_host] - user = current_application_settings[:metrics_username] - pw = current_application_settings[:metrics_password] - port = current_application_settings[:metrics_port] + @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] InfluxDB::Client. - new(udp: { host: host, port: port }, username: user, password: pw) + new(udp: { host: host, port: port }) end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 06fc2f25948..face1921d2e 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -106,23 +106,41 @@ module Gitlab if type == :instance target = mod label = "#{mod.name}##{name}" + method = mod.instance_method(name) else target = mod.singleton_class label = "#{mod.name}.#{name}" + method = mod.method(name) end + # Some code out there (e.g. the "state_machine" Gem) checks the arity of + # a method to make sure it only passes arguments when the method expects + # any. If we were to always overwrite a method to take an `*args` + # signature this would break things. As a result we'll make sure the + # generated method _only_ accepts regular arguments if the underlying + # method also accepts them. + if method.arity == 0 + args_signature = '&block' + else + args_signature = '*args, &block' + end + + send_signature = "__send__(#{alias_name.inspect}, #{args_signature})" + target.class_eval <<-EOF, __FILE__, __LINE__ + 1 alias_method #{alias_name.inspect}, #{name.inspect} - def #{name}(*args, &block) + def #{name}(#{args_signature}) trans = Gitlab::Metrics::Instrumentation.transaction if trans start = Time.now - retval = __send__(#{alias_name.inspect}, *args, &block) + retval = #{send_signature} duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + trans.increment(:method_duration, duration) + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, { duration: duration }, method: #{label.inspect}) @@ -130,7 +148,7 @@ module Gitlab retval else - __send__(#{alias_name.inspect}, *args, &block) + #{send_signature} end end EOF diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 753008df99a..7ea9555cc8c 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -17,11 +17,8 @@ module Gitlab # Returns a Hash in a format that can be directly written to InfluxDB. def to_hash { - series: @series, - tags: @tags.merge( - hostname: Metrics.hostname, - process_type: Sidekiq.server? ? 'sidekiq' : 'rails' - ), + series: @series, + tags: @tags, values: @values, timestamp: @created_at.to_i * 1_000_000_000 } diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb deleted file mode 100644 index fe97d7a0534..00000000000 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Metrics - # Class for producing SQL queries with sensitive data stripped out. - class ObfuscatedSQL - REPLACEMENT = / - \d+(\.\d+)? # integers, floats - | '.+?' # single quoted strings - | \/.+?(?<!\\)\/ # regexps (including escaped slashes) - /x - - MYSQL_REPLACEMENTS = / - ".+?" # double quoted strings - /x - - # Regex to replace consecutive placeholders with a single one indicating - # the length. This can be useful when a "IN" statement uses thousands of - # IDs (storing this would just be a waste of space). - CONSECUTIVE = /(\?(\s*,\s*)?){2,}/ - - # sql - The raw SQL query as a String. - def initialize(sql) - @sql = sql - end - - # Returns a new, obfuscated SQL query. - def to_s - regex = REPLACEMENT - - if Gitlab::Database.mysql? - regex = Regexp.union(regex, MYSQL_REPLACEMENTS) - end - - sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match| - "#{match.count(',') + 1} values" - end - - # InfluxDB escapes double quotes upon output, so lets get rid of them - # whenever we can. - if Gitlab::Database.postgresql? - sql = sql.delete('"') - end - - sql.tr("\n", ' ') - end - end - end -end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 5c0587c4c51..6f179789d3e 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -32,17 +32,15 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.add_tag(:request_method, env['REQUEST_METHOD']) - trans.add_tag(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_method, env['REQUEST_METHOD']) trans end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - label = "#{controller.class.name}##{controller.action_name}" - - trans.add_tag(:action, label) + controller = env[CONTROLLER_KEY] + trans.action = "#{controller.class.name}##{controller.action_name}" end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 998578e1c0a..fc709222a9b 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -7,9 +7,14 @@ module Gitlab # statistics, etc. class Sampler # interval - The sampling interval in seconds. - def initialize(interval = 15) - @interval = interval - @metrics = [] + def initialize(interval = Metrics.settings[:sample_interval]) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + @last_step = nil + + @metrics = [] @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) @@ -26,7 +31,7 @@ module Gitlab Thread.current.abort_on_exception = true loop do - sleep(@interval) + sleep(sleep_interval) sample end @@ -50,12 +55,11 @@ module Gitlab end def sample_memory_usage - @metrics << Metric.new('memory_usage', value: System.memory_usage) + add_metric('memory_usage', value: System.memory_usage) end def sample_file_descriptors - @metrics << Metric. - new('file_descriptors', value: System.file_descriptor_count) + add_metric('file_descriptors', value: System.file_descriptor_count) end if Metrics.mri? @@ -69,7 +73,7 @@ module Gitlab counts['Symbol'] = Symbol.all_symbols.length counts.each do |name, count| - @metrics << Metric.new('object_counts', { count: count }, type: name) + add_metric('object_counts', { count: count }, type: name) end end else @@ -91,7 +95,34 @@ module Gitlab stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - @metrics << Metric.new('gc_statistics', stats) + add_metric('gc_statistics', stats) + end + + def add_metric(series, values, tags = {}) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def sidekiq? + Sidekiq.server? + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end end end end diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ad441decfa2..fd98aa3412e 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,19 +5,14 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - trans = Transaction.new + trans = Transaction.new("#{worker.class.name}#perform") begin trans.run { yield } ensure - tag_worker(trans, worker) trans.finish end end - - def tag_worker(trans, worker) - trans.add_tag(:action, "#{worker.class.name}#perform") - end end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 7e0dcf99d92..2e9dd4645e3 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -19,6 +19,7 @@ module Gitlab values = values_for(event) tags = tags_for(event) + current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end @@ -32,16 +33,8 @@ module Gitlab def tags_for(event) path = relative_path(event.payload[:identifier]) - tags = { view: path } - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags + { view: path } end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d947c128ce2..8008b3bc895 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -1,44 +1,18 @@ module Gitlab module Metrics module Subscribers - # Class for tracking raw SQL queries. - # - # Queries are obfuscated before being logged to ensure no private data is - # exposed via InfluxDB/Grafana. + # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber attach_to :active_record - SERIES = 'sql_queries' - def sql(event) return unless current_transaction - values = values_for(event) - tags = tags_for(event) - - current_transaction.add_metric(SERIES, values, tags) + current_transaction.increment(:sql_duration, event.duration) end private - def values_for(event) - { duration: event.duration } - end - - def tags_for(event) - sql = ObfuscatedSQL.new(event.payload[:sql]).to_s - tags = { sql: sql } - - file, line = Metrics.last_relative_application_frame - - if file and line - tags[:file] = file - tags[:line] = line - end - - tags - end - def current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index a61dbd989e7..2578ddc49f4 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,45 +4,64 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - SERIES = 'transactions' + attr_reader :tags, :values - attr_reader :uuid, :tags + attr_accessor :action def self.current Thread.current[THREAD_KEY] end - # name - The name of this transaction as a String. - def initialize + # action - A String describing the action performed, usually the class + # plus method name. + def initialize(action = nil) @metrics = [] - @uuid = SecureRandom.uuid @started_at = nil @finished_at = nil - @tags = {} + @values = Hash.new(0) + @tags = {} + @action = action + + @memory_before = 0 + @memory_after = 0 end def duration @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 end + def allocated_memory + @memory_after - @memory_before + end + def run Thread.current[THREAD_KEY] = self - @started_at = Time.now + @memory_before = System.memory_usage + @started_at = Time.now yield ensure - @finished_at = Time.now + @memory_after = System.memory_usage + @finished_at = Time.now Thread.current[THREAD_KEY] = nil end def add_metric(series, values, tags = {}) - tags = tags.merge(transaction_id: @uuid) + prefix = sidekiq? ? 'sidekiq_' : 'rails_' + + @metrics << Metric.new("#{prefix}#{series}", values, tags) + end + + def increment(name, value) + @values[name] += value + end - @metrics << Metric.new(series, values, tags) + def set(name, value) + @values[name] = value end def add_tag(key, value) @@ -55,11 +74,29 @@ module Gitlab end def track_self - add_metric(SERIES, { duration: duration }, @tags) + values = { duration: duration, allocated_memory: allocated_memory } + + @values.each do |name, value| + values[name] = value + end + + add_metric('transactions', values, @tags) end def submit - Metrics.submit_metrics(@metrics.map(&:to_hash)) + metrics = @metrics.map do |metric| + hash = metric.to_hash + + hash[:tags][:action] ||= @action if @action + + hash + end + + Metrics.submit_metrics(metrics) + end + + def sidekiq? + Sidekiq.server? end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb new file mode 100644 index 00000000000..50b0dd32380 --- /dev/null +++ b/lib/gitlab/middleware/go.rb @@ -0,0 +1,50 @@ +# A dumb middleware that returns a Go HTML document if the go-get=1 query string +# is used irrespective if the namespace/project exists +module Gitlab + module Middleware + class Go + def initialize(app) + @app = app + end + + def call(env) + request = Rack::Request.new(env) + + if go_request?(request) + render_go_doc(request) + else + @app.call(env) + end + end + + private + + def render_go_doc(request) + body = go_body(request) + response = Rack::Response.new(body, 200, { 'Content-Type' => 'text/html' }) + response.finish + end + + def go_request?(request) + request["go-get"].to_i == 1 && request.env["PATH_INFO"].present? + end + + def go_body(request) + base_url = Gitlab.config.gitlab.url + # Go subpackages may be in the form of namespace/project/path1/path2/../pathN + # We can just ignore the paths and leave the namespace/project + path_info = request.env["PATH_INFO"] + path_info.sub!(/^\//, '') + project_path = path_info.split('/').first(2).join('/') + request_url = URI.join(base_url, project_path) + domain_path = strip_url(request_url.to_s) + + "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n"; + end + + def strip_url(url) + url.gsub(/\Ahttps?:\/\//, '') + end + end + end +end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index ea6b0ee796d..71cf6a0d886 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -53,13 +53,10 @@ module Gitlab object_kind: "note", user: user.hook_attrs, project_id: project.id, - repository: { - name: project.name, - url: project.url_to_repo, - description: project.description, - homepage: project.web_url, - }, - object_attributes: note.hook_attrs + project: project.hook_attrs, + object_attributes: note.hook_attrs, + # DEPRECATED + repository: project.hook_attrs.slice(:name, :url, :description, :homepage) } base_data[:object_attributes][:url] = diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index ba31599432b..36e5c2670bb 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -32,6 +32,10 @@ module Gitlab @password ||= Gitlab::Utils.force_utf8(Devise.friendly_token[0, 8].downcase) end + def has_email? + get_info(:email).present? + end + private def info @@ -46,8 +50,8 @@ module Gitlab def username_and_email @username_and_email ||= begin - username = get_info(:username) || get_info(:nickname) - email = get_info(:email) + username = get_info(:username).presence || get_info(:nickname).presence + email = get_info(:email).presence username ||= generate_username(email) if email email ||= generate_temporarily_email(username) if username diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index f1a362f5303..832fb08a526 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -26,7 +26,7 @@ module Gitlab gl_user.try(:valid?) end - def save + def save(provider = 'OAuth') unauthorized_to_create unless gl_user if needs_blocking? @@ -36,10 +36,10 @@ module Gitlab gl_user.save! end - log.info "(OAuth) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e - log.info "(OAuth) Error saving user: #{gl_user.errors.full_messages}" + log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}" return self, e.record.errors end @@ -105,13 +105,18 @@ module Gitlab end def signup_enabled? - Gitlab.config.omniauth.allow_single_sign_on + providers = Gitlab.config.omniauth.allow_single_sign_on + if providers.is_a?(Array) + providers.include?(auth_hash.provider) + else + providers + end end def block_after_signup? if creating_linked_ldap_user? ldap_config.block_auto_created_users - else + else Gitlab.config.omniauth.block_auto_created_users end end @@ -135,15 +140,18 @@ module Gitlab def user_attributes # Give preference to LDAP for sensitive information when creating a linked account if creating_linked_ldap_user? - username = ldap_person.username - email = ldap_person.email.first - else - username = auth_hash.username - email = auth_hash.email + username = ldap_person.username.presence + email = ldap_person.email.first.presence end - + + username ||= auth_hash.username + email ||= auth_hash.email + + name = auth_hash.name + name = ::Namespace.clean_path(username) if name.strip.empty? + { - name: auth_hash.name, + name: name, username: ::Namespace.clean_path(username), email: email, password: auth_hash.password, diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb new file mode 100644 index 00000000000..746ec283330 --- /dev/null +++ b/lib/gitlab/other_markup.rb @@ -0,0 +1,24 @@ +module Gitlab + # Parser/renderer for markups without other special support code. + module OtherMarkup + + # Public: Converts the provided markup into HTML. + # + # input - the source text in a markup format + # context - a Hash with the template context: + # :commit + # :project + # :project_wiki + # :requested_path + # :ref + # + def self.render(file_name, input, context) + html = GitHub::Markup.render(file_name, input). + force_encoding(input.encoding) + + html = Banzai.post_process(html, context) + + html.html_safe + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 70de6a74e76..71c5b6801fb 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,8 +2,9 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(project_id, query, repository_ref = nil) - @project = Project.find(project_id) + def initialize(current_user, project, query, repository_ref = nil) + @current_user = current_user + @project = project @repository_ref = if repository_ref.present? repository_ref else @@ -73,7 +74,7 @@ module Gitlab end def notes - Note.where(project_id: limit_project_ids).user.search(query).order('updated_at DESC') + project.notes.user.search(query).order('updated_at DESC') end def commits @@ -84,8 +85,8 @@ module Gitlab end end - def limit_project_ids - [project.id] + def project_ids_relation + project end end end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index 4f9cdef3869..97d1edab9c1 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -22,6 +22,8 @@ module Gitlab # } # def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + commits = Array(commits) + # Total commits count commits_count = commits.size @@ -47,25 +49,21 @@ module Gitlab user_id: user.id, user_name: user.name, user_email: user.email, + user_avatar: user.avatar_url, project_id: project.id, - repository: { - name: project.name, - url: project.url_to_repo, - description: project.description, - homepage: project.web_url, - git_http_url: project.http_url_to_repo, - git_ssh_url: project.ssh_url_to_repo, - visibility_level: project.visibility_level - }, + project: project.hook_attrs, commits: commit_attrs, - total_commits_count: commits_count + total_commits_count: commits_count, + # DEPRECATED + repository: project.hook_attrs.slice(:name, :url, :description, :homepage, + :git_http_url, :git_ssh_url, :visibility_level) } data end # This method provide a sample data generated with - # existing project and commits to test web hooks + # existing project and commits to test webhooks def build_sample(project, user) commits = project.repository.commits(project.default_branch, nil, 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" diff --git a/lib/gitlab/redis_config.rb b/lib/gitlab/redis_config.rb new file mode 100644 index 00000000000..4949c6db539 --- /dev/null +++ b/lib/gitlab/redis_config.rb @@ -0,0 +1,30 @@ +module Gitlab + class RedisConfig + attr_reader :url + + def self.url + new.url + end + + def self.redis_store_options + url = new.url + redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) + # Redis::Store does not handle Unix sockets well, so let's do it for them + redis_uri = URI.parse(url) + if redis_uri.scheme == 'unix' + redis_config_hash[:path] = redis_uri.path + end + redis_config_hash + end + + def initialize(rails_env=nil) + rails_env ||= Rails.env + config_file = File.expand_path('../../../config/resque.yml', __FILE__) + + @url = "redis://localhost:6379" + if File.exists?(config_file) + @url =YAML.load_file(config_file)[rails_env] + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index be795649e59..4d830aa45e1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor @@ -19,7 +17,7 @@ module Gitlab super(text, context.merge(project: project)) end - %i(user label merge_request snippet commit commit_range).each do |type| + %i(user label milestone merge_request snippet commit commit_range).each do |type| define_method("#{type}s") do @references[type] ||= references(type, reference_context) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 53ab2686b43..ace906a6f59 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -34,29 +34,29 @@ module Gitlab def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git)\z/.freeze + @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end def project_path_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.git'" \ + "Cannot start with '-', end in '.git' or end in '.atom'" \ end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.]*\z/.freeze + @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'. " end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/]*\z/.freeze + @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze end def file_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb new file mode 100644 index 00000000000..b1e30110ef5 --- /dev/null +++ b/lib/gitlab/saml/user.rb @@ -0,0 +1,47 @@ +# SAML extension for User model +# +# * Find GitLab user based on SAML uid and provider +# * Create new user from SAML data +# +module Gitlab + module Saml + class User < Gitlab::OAuth::User + + def save + super('SAML') + end + + def gl_user + @user ||= find_by_uid_and_provider + + if auto_link_ldap_user? + @user ||= find_or_create_ldap_user + end + + if auto_link_saml_enabled? + @user ||= find_by_email + end + + if signup_enabled? + @user ||= build_new_user + end + + @user + end + + def find_by_email + if auth_hash.has_email? + user = ::User.find_by(email: auth_hash.email.downcase) + user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) if user + user + end + end + + protected + + def auto_link_saml_enabled? + Gitlab.config.omniauth.auto_link_saml_user + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 2ab2d4af797..f8ab2b1f09e 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,13 +1,14 @@ module Gitlab class SearchResults - attr_reader :query + attr_reader :current_user, :query - # Limit search results by passed project ids + # Limit search results by passed projects # It allows us to search only for projects user has access to - attr_reader :limit_project_ids + attr_reader :limit_projects - def initialize(limit_project_ids, query) - @limit_project_ids = limit_project_ids || Project.all + def initialize(current_user, limit_projects, query) + @current_user = current_user + @limit_projects = limit_projects || Project.all @query = Shellwords.shellescape(query) if query.present? end @@ -27,7 +28,8 @@ module Gitlab end def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count + @total_count ||= projects_count + issues_count + merge_requests_count + + milestones_count end def projects_count @@ -53,27 +55,29 @@ module Gitlab private def projects - Project.where(id: limit_project_ids).search(query) + limit_projects.search(query) end def issues - issues = Issue.where(project_id: limit_project_ids) + issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) + if query =~ /#(\d+)\z/ issues = issues.where(iid: $1) else issues = issues.full_search(query) end + issues.order('updated_at DESC') end def milestones - milestones = Milestone.where(project_id: limit_project_ids) + milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) milestones.order('updated_at DESC') end def merge_requests - merge_requests = MergeRequest.in_projects(limit_project_ids) + merge_requests = MergeRequest.in_projects(project_ids_relation) if query =~ /[#!](\d+)\z/ merge_requests = merge_requests.where(iid: $1) else @@ -89,5 +93,9 @@ module Gitlab def per_page 20 end + + def project_ids_relation + limit_projects.select(:id).reorder(nil) + end end end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 938219efdb2..e0e74ff8359 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -1,18 +1,20 @@ module Gitlab class SnippetSearchResults < SearchResults - attr_reader :limit_snippet_ids + include SnippetsHelper - def initialize(limit_snippet_ids, query) - @limit_snippet_ids = limit_snippet_ids + attr_reader :limit_snippets + + def initialize(limit_snippets, query) + @limit_snippets = limit_snippets @query = query end def objects(scope, page = nil) case scope when 'snippet_titles' - Kaminari.paginate_array(snippet_titles).page(page).per(per_page) + snippet_titles.page(page).per(per_page) when 'snippet_blobs' - Kaminari.paginate_array(snippet_blobs).page(page).per(per_page) + snippet_blobs.page(page).per(per_page) else super end @@ -33,99 +35,15 @@ module Gitlab private def snippet_titles - Snippet.where(id: limit_snippet_ids).search(query).order('updated_at DESC') + limit_snippets.search(query).order('updated_at DESC') end def snippet_blobs - search = Snippet.where(id: limit_snippet_ids).search_code(query) - search = search.order('updated_at DESC').to_a - snippets = [] - search.each { |e| snippets << chunk_snippet(e) } - snippets + limit_snippets.search_code(query).order('updated_at DESC') end def default_scope 'snippet_blobs' end - - # Get an array of line numbers surrounding a matching - # line, bounded by min/max. - # - # @returns Array of line numbers - def bounded_line_numbers(line, min, max) - lower = line - surrounding_lines > min ? line - surrounding_lines : min - upper = line + surrounding_lines < max ? line + surrounding_lines : max - (lower..upper).to_a - end - - # Returns a sorted set of lines to be included in a snippet preview. - # This ensures matching adjacent lines do not display duplicated - # surrounding code. - # - # @returns Array, unique and sorted. - def matching_lines(lined_content) - used_lines = [] - lined_content.each_with_index do |line, line_number| - used_lines.concat bounded_line_numbers( - line_number, - 0, - lined_content.size - ) if line.include?(query) - end - - used_lines.uniq.sort - end - - # 'Chunkify' entire snippet. Splits the snippet data into matching lines + - # surrounding_lines() worth of unmatching lines. - # - # @returns a hash with {snippet_object, snippet_chunks:{data,start_line}} - def chunk_snippet(snippet) - lined_content = snippet.content.split("\n") - used_lines = matching_lines(lined_content) - - snippet_chunk = [] - snippet_chunks = [] - snippet_start_line = 0 - last_line = -1 - - # Go through each used line, and add consecutive lines as a single chunk - # to the snippet chunk array. - used_lines.each do |line_number| - if last_line < 0 - # Start a new chunk. - snippet_start_line = line_number - snippet_chunk << lined_content[line_number] - elsif last_line == line_number - 1 - # Consecutive line, continue chunk. - snippet_chunk << lined_content[line_number] - else - # Non-consecutive line, add chunk to chunk array. - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Start a new chunk. - snippet_chunk = [lined_content[line_number]] - snippet_start_line = line_number - end - last_line = line_number - end - # Add final chunk to chunk array - snippet_chunks << { - data: snippet_chunk.join("\n"), - start_line: snippet_start_line + 1 - } - - # Return snippet with chunk array - { snippet_object: snippet, snippet_chunks: snippet_chunks } - end - - # Defines how many unmatching lines should be - # included around the matching lines in a snippet - def surrounding_lines - 3 - end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 4885baf9526..d1b42c1f9b9 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -3,7 +3,7 @@ module Gitlab def self.allowed?(user) return false if user.blocked? - if user.requires_ldap_check? + if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb new file mode 100644 index 00000000000..c3ddd4c2680 --- /dev/null +++ b/lib/gitlab/workhorse.rb @@ -0,0 +1,40 @@ +require 'base64' +require 'json' + +module Gitlab + class Workhorse + SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' + + class << self + def send_git_blob(repository, blob) + params = { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id, + } + + [ + SEND_DATA_HEADER, + "git-blob:#{encode(params)}", + ] + end + + def send_git_archive(project, ref, format) + format ||= 'tar.gz' + format.downcase! + params = project.repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + raise "Repository or ref not found" if params.empty? + + [ + SEND_DATA_HEADER, + "git-archive:#{encode(params)}", + ] + end + + protected + + def encode(hash) + Base64.urlsafe_encode64(JSON.dump(hash)) + end + end + end +end |