diff options
Diffstat (limited to 'lib/gitlab')
79 files changed, 1214 insertions, 290 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 0b9c2e730f9..1a22ad9acf5 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,7 +4,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index 51b1df9ecbd..c94bfc0e65f 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -66,8 +66,17 @@ module Gitlab def self.urls @urls ||= begin path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + # Construct the full asset path ourselves because + # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds + # of entries since it has to do a lot of extra work (e.g. regexps). prefix = Gitlab::Application.config.assets.prefix digest = Gitlab::Application.config.assets.digest + base = + if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root + Gitlab::Application.config.relative_url_root + else + '' + end JSON.parse(File.read(path)).map do |hash| if digest @@ -76,7 +85,7 @@ module Gitlab fname = hash['unicode'] end - { name: hash['name'], path: "#{prefix}/#{fname}.png" } + { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index ab7b811c5d8..478f145bfed 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -8,7 +8,6 @@ module Grack end class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env def call(env) @@ -22,7 +21,7 @@ module Grack # Need this if under RELATIVE_URL_ROOT unless Gitlab.config.gitlab.relative_url_root.empty? # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'') + @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') else @env['PATH_INFO'] = @request.path end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 997a22779a0..d62bc50ce78 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -41,7 +41,8 @@ module Gitlab def highlighted_lines @blob.load_all_data!(repository) - @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines + @highlighted_lines ||= + Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines end def project diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index adfd097736e..e6cc1529760 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -7,7 +7,8 @@ module Gitlab ## # Temporary delegations that should be removed after refactoring # - delegate :before_script, to: :@global + delegate :before_script, :image, :services, :after_script, :variables, + :stages, :cache, to: :@global def initialize(config) @config = Loader.new(config).load! diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/node/boolean.rb new file mode 100644 index 00000000000..84b03ee7832 --- /dev/null +++ b/lib/gitlab/ci/config/node/boolean.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a boolean value. + # + class Boolean < Entry + include Validatable + + validations do + validates :config, boolean: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/node/cache.rb new file mode 100644 index 00000000000..cdf8ba2e35d --- /dev/null +++ b/lib/gitlab/ci/config/node/cache.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a cache configuration + # + class Cache < Entry + include Configurable + + node :key, Node::Key, + description: 'Cache key used to define a cache affinity.' + + node :untracked, Node::Boolean, + description: 'Cache all untracked files.' + + node :paths, Node::Paths, + description: 'Specify which paths should be cached across builds.' + + validations do + validates :config, allowed_keys: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index 374ff71d0f5..37936fc8242 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -19,35 +19,45 @@ module Gitlab included do validations do - validates :config, hash: true + validates :config, type: Hash end end private def create_node(key, factory) - factory.with(value: @config[key], key: key) - factory.nullify! unless @config.has_key?(key) + factory.with(value: @config[key], key: key, parent: self) + factory.create! end class_methods do def nodes - Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] end private - def allow_node(symbol, entry_class, metadata) + def node(symbol, entry_class, metadata) factory = Node::Factory.new(entry_class) .with(description: metadata[:description]) - define_method(symbol) do - raise Entry::InvalidError unless valid? - @nodes[symbol].try(:value) - end + (@nodes ||= {}).merge!(symbol.to_sym => factory) + end - (@allowed_nodes ||= {}).merge!(symbol => factory) + def helpers(*nodes) + nodes.each do |symbol| + define_method("#{symbol}_defined?") do + @nodes[symbol].try(:defined?) + end + + define_method("#{symbol}_value") do + raise Entry::InvalidError unless valid? + @nodes[symbol].try(:value) + end + + alias_method symbol.to_sym, "#{symbol}_value".to_sym + end end end end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index f044ef965e9..9e79e170a4f 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -9,7 +9,7 @@ module Gitlab class InvalidError < StandardError; end attr_reader :config - attr_accessor :key, :description + attr_accessor :key, :parent, :description def initialize(config) @config = config @@ -34,8 +34,8 @@ module Gitlab self.class.nodes.none? end - def key - @key || self.class.name.demodulize.underscore + def ancestors + @parent ? @parent.ancestors + [@parent] : [] end def valid? @@ -43,12 +43,23 @@ module Gitlab end def errors - @validator.full_errors + - nodes.map(&:errors).flatten + @validator.messages + nodes.flat_map(&:errors) end def value - raise NotImplementedError + if leaf? + @config + else + defined = @nodes.select { |_key, value| value.defined? } + Hash[defined.map { |key, node| [key, node.value] }] + end + end + + def defined? + true + end + + def self.default end def self.nodes diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 025ae40ef94..5919a283283 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -5,13 +5,11 @@ module Gitlab ## # Factory class responsible for fabricating node entry objects. # - # It uses Fluent Interface pattern to set all necessary attributes. - # class Factory class InvalidFactory < StandardError; end - def initialize(entry_class) - @entry_class = entry_class + def initialize(node) + @node = node @attributes = {} end @@ -20,17 +18,27 @@ module Gitlab self end - def nullify! - @entry_class = Node::Null - self - end - def create! raise InvalidFactory unless @attributes.has_key?(:value) - @entry_class.new(@attributes[:value]).tap do |entry| - entry.description = @attributes[:description] + fabricate.tap do |entry| entry.key = @attributes[:key] + entry.parent = @attributes[:parent] + entry.description = @attributes[:description] + end + end + + private + + def fabricate + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @attributes[:value].nil? + Node::Undefined.new(@node) + else + @node.new(@attributes[:value]) end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index 044603423d5..f92e1eccbcf 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -9,8 +9,36 @@ module Gitlab class Global < Entry include Configurable - allow_node :before_script, Script, + node :before_script, Node::Script, description: 'Script that will be executed before each job.' + + node :image, Node::Image, + description: 'Docker image that will be used to execute jobs.' + + node :services, Node::Services, + description: 'Docker images that will be linked to the container.' + + node :after_script, Node::Script, + description: 'Script that will be executed after each job.' + + node :variables, Node::Variables, + description: 'Environment variables that will be used.' + + node :stages, Node::Stages, + description: 'Configuration of stages for this pipeline.' + + node :types, Node::Stages, + description: 'Deprecated: stages for this pipeline.' + + node :cache, Node::Cache, + description: 'Configure caching between build jobs.' + + helpers :before_script, :image, :services, :after_script, + :variables, :stages, :types, :cache + + def stages + stages_defined? ? stages_value : types_value + end end end end diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/node/image.rb new file mode 100644 index 00000000000..5d3c7c5eab0 --- /dev/null +++ b/lib/gitlab/ci/config/node/image.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a Docker image. + # + class Image < Entry + include Validatable + + validations do + validates :config, type: String + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/node/key.rb new file mode 100644 index 00000000000..f8b461ca098 --- /dev/null +++ b/lib/gitlab/ci/config/node/key.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a key. + # + class Key < Entry + include Validatable + + validations do + validates :config, key: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 4f590f6bec8..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents a configuration entry that is not being used - # in configuration file. - # - # This implements Null Object pattern. - # - class Null < Entry - def value - nil - end - - def validate! - nil - end - - def method_missing(*) - nil - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/node/paths.rb new file mode 100644 index 00000000000..3c6d3a52966 --- /dev/null +++ b/lib/gitlab/ci/config/node/paths.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an array of paths. + # + class Paths < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb index c044f5c5e71..39328f0fade 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/node/script.rb @@ -5,21 +5,12 @@ module Gitlab ## # Entry that represents a script. # - # Each element in the value array is a command that will be executed - # by GitLab Runner. Currently we concatenate these commands with - # new line character as a separator, what is compatible with - # implementation in Runner. - # class Script < Entry include Validatable validations do validates :config, array_of_strings: true end - - def value - @config.join("\n") - end end end end diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/node/services.rb new file mode 100644 index 00000000000..481e2b66adc --- /dev/null +++ b/lib/gitlab/ci/config/node/services.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration of Docker services. + # + class Services < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/node/stages.rb new file mode 100644 index 00000000000..b1fe45357ff --- /dev/null +++ b/lib/gitlab/ci/config/node/stages.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a configuration for pipeline stages. + # + class Stages < Entry + include Validatable + + validations do + validates :config, array_of_strings: true + end + + def self.default + %w[build test deploy] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb new file mode 100644 index 00000000000..699605e1e3a --- /dev/null +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -0,0 +1,30 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an undefined entry node. + # + # It takes original entry class as configuration and returns default + # value of original entry as self value. + # + # + class Undefined < Entry + include Validatable + + validations do + validates :config, type: Class + end + + def value + @config.default + end + + def defined? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb index 02edc9219c3..758a6cf4356 100644 --- a/lib/gitlab/ci/config/node/validator.rb +++ b/lib/gitlab/ci/config/node/validator.rb @@ -11,15 +11,29 @@ module Gitlab @node = node end - def full_errors + def messages errors.full_messages.map do |error| - "#{@node.key} #{error}".humanize + "#{location} #{error}".downcase end end def self.name 'Validator' end + + def unknown_keys + return [] unless config.is_a?(Hash) + + config.keys - @node.class.nodes.keys + end + + private + + def location + predecessors = ancestors.map(&:key).compact + current = key || @node.class.name.demodulize.underscore + predecessors.append(current).join(':') + end end end end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb index dc9cdb9a220..7b2f57990b5 100644 --- a/lib/gitlab/ci/config/node/validators.rb +++ b/lib/gitlab/ci/config/node/validators.rb @@ -3,6 +3,16 @@ module Gitlab class Config module Node module Validators + class AllowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + if record.unknown_keys.any? + unknown_list = record.unknown_keys.join(', ') + record.errors.add(:config, + "contains unknown keys: #{unknown_list}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -13,10 +23,43 @@ module Gitlab end end - class HashValidator < ActiveModel::EachValidator + class BooleanValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_boolean(value) + record.errors.add(attribute, 'should be a boolean value') + end + end + end + + class KeyValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_string(value) + record.errors.add(attribute, 'should be a string or symbol') + end + end + end + + class TypeValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + type = options[:with] + raise unless type.is_a?(Class) + + unless value.is_a?(type) + record.errors.add(attribute, "should be a #{type.name}") + end + end + end + + class VariablesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + def validate_each(record, attribute, value) - unless value.is_a?(Hash) - record.errors.add(attribute, 'should be a configuration entry hash') + unless validate_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs') end end end diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/node/variables.rb new file mode 100644 index 00000000000..5f813f81f55 --- /dev/null +++ b/lib/gitlab/ci/config/node/variables.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents environment variables. + # + class Variables < Entry + include Validatable + + validations do + validates :config, variables: true + end + + def self.default + {} + end + end + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 28c34429c1f..ffc1814b29d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,10 +9,14 @@ module Gitlab end def ensure_application_settings! - settings = ::ApplicationSetting.cached + if connect_to_db? + begin + settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + settings = ::ApplicationSetting.last + end - if !settings && connect_to_db? - settings = ::ApplicationSetting.current settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end @@ -44,6 +48,7 @@ module Gitlab akismet_enabled: false, repository_checks_enabled: true, container_registry_token_expire_delay: 5, + user_default_external: false, ) end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb new file mode 100644 index 00000000000..8406ca4269c --- /dev/null +++ b/lib/gitlab/diff/diff_refs.rb @@ -0,0 +1,36 @@ +module Gitlab + module Diff + class DiffRefs + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(base_sha:, start_sha: base_sha, head_sha:) + @base_sha = base_sha + @start_sha = start_sha + @head_sha = head_sha + end + + def ==(other) + other.is_a?(self.class) && + base_sha == other.base_sha && + start_sha == other.start_sha && + head_sha == other.head_sha + end + + # There is only one case in which we will have `start_sha` and `head_sha`, + # but not `base_sha`, which is when a diff is generated between an + # orphaned branch and another branch, which means there _is_ no base, but + # we're still able to highlight it, and to create diff notes, which are + # the primary things `DiffRefs` are used for. + # `DiffRefs` are "complete" when they have `start_sha` and `head_sha`, + # because `base_sha` can always be derived from this, to return an actual + # sha, or `nil`. + # We have `base_sha` directly available on `DiffRefs` because it's faster# + # than having to look it up in the repo every time. + def complete? + start_sha && head_sha + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d2e85cabf72..b0c50edba59 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,47 +1,83 @@ module Gitlab module Diff class File - attr_reader :diff, :diff_refs + attr_reader :diff, :repository, :diff_refs delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, to: :diff, prefix: false + :old_path, :new_path, :a_mode, :b_mode, + :submodule?, :too_large?, to: :diff, prefix: false - def initialize(diff, diff_refs) + def initialize(diff, repository:, diff_refs: nil) @diff = diff + @repository = repository @diff_refs = diff_refs end + def position(line) + return unless diff_refs + + Position.new( + old_path: old_path, + new_path: new_path, + old_line: line.old_line, + new_line: line.new_line, + diff_refs: diff_refs + ) + end + + def line_code(line) + return if line.meta? + + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def line_for_line_code(code) + diff_lines.find { |line| line_code(line) == code } + end + + def line_for_position(pos) + diff_lines.find { |line| position(line) == pos } + end + + def position_for_line_code(code) + line = line_for_line_code(code) + position(line) if line + end + + def line_code_for_position(pos) + line = line_for_position(pos) + line_code(line) if line + end + + def content_commit + return unless diff_refs + + repository.commit(deleted_file ? old_ref : new_ref) + end + def old_ref - diff_refs[0] if diff_refs + diff_refs.try(:base_sha) end def new_ref - diff_refs[1] if diff_refs + diff_refs.try(:head_sha) end - # Array of Gitlab::DIff::Line objects + # Array of Gitlab::Diff::Line objects def diff_lines - @lines ||= parser.parse(raw_diff.each_line).to_a - end - - def too_large? - diff.too_large? + @lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a end def highlighted_diff_lines - Gitlab::Diff::Highlight.new(self).highlight + @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end def parallel_diff_lines - Gitlab::Diff::ParallelDiff.new(self).parallelize + @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end def mode_changed? - !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode) - end - - def parser - Gitlab::Diff::Parser.new + a_mode && b_mode && a_mode != b_mode end def raw_diff @@ -53,17 +89,15 @@ module Gitlab end def prev_line(index) - if index > 0 - diff_lines[index - 1] - end + diff_lines[index - 1] if index > 0 + end + + def paths + [old_path, new_path].compact end def file_path - if diff.new_path.present? - diff.new_path - elsif diff.old_path.present? - diff.old_path - end + new_path.presence || old_path end def added_lines @@ -73,6 +107,21 @@ module Gitlab def removed_lines diff_lines.count(&:removed?) end + + def old_blob(commit = content_commit) + return unless commit + + parent_id = commit.parent_id + return unless parent_id + + repository.blob_at(parent_id, old_path) + end + + def blob(commit = content_commit) + return unless commit + + repository.blob_at(commit.id, file_path) + end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 9429b3ff88d..649a265a02c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,11 +1,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines + attr_reader :diff_file, :diff_lines, :raw_lines, :repository delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff - def initialize(diff_lines) + def initialize(diff_lines, repository: nil) + @repository = repository + if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @diff_lines = @diff_file.diff_lines @@ -19,7 +21,7 @@ module Gitlab @diff_lines.map.with_index do |diff_line, i| diff_line = diff_line.dup # ignore highlighting for "match" lines - next diff_line if diff_line.type == 'match' || diff_line.type == 'nonewline' + next diff_line if diff_line.meta? rich_line = highlight_line(diff_line) || diff_line.text @@ -40,12 +42,12 @@ module Gitlab line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' - case diff_line.type - when 'new', nil - rich_line = new_lines[diff_line.new_pos - 1] - when 'old' - rich_line = old_lines[diff_line.old_pos - 1] - end + rich_line = + if diff_line.unchanged? || diff_line.added? + new_lines[diff_line.new_pos - 1] + elsif diff_line.removed? + old_lines[diff_line.old_pos - 1] + end # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. @@ -58,19 +60,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:old)) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(*processing_args(:new)) - end - - def processing_args(version) - ref = send("diff_#{version}_ref") - path = send("diff_#{version}_path") - - [ref.project.repository, ref.id, path] + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 03730b435ad..c6189d660c2 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -9,6 +9,18 @@ module Gitlab @old_pos, @new_pos = old_pos, new_pos end + def old_line + old_pos unless added? || meta? + end + + def new_line + new_pos unless removed? || meta? + end + + def unchanged? + type.nil? + end + def added? type == 'new' end @@ -16,6 +28,10 @@ module Gitlab def removed? type == 'old' end + + def meta? + type == 'match' || type == 'nonewline' + end end end end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb new file mode 100644 index 00000000000..576a761423e --- /dev/null +++ b/lib/gitlab/diff/line_mapper.rb @@ -0,0 +1,64 @@ +# When provided a diff for a specific file, maps old line numbers to new line +# numbers and back, to find out where a specific line in a file was moved by the +# changes. +module Gitlab + module Diff + class LineMapper + attr_accessor :diff_file + + def initialize(diff_file) + @diff_file = diff_file + end + + # Find new line number for old line number. + def old_to_new(old_line) + map_line_number(old_line, from: :old_line, to: :new_line) + end + + # Find old line number for new line number. + def new_to_old(new_line) + map_line_number(new_line, from: :new_line, to: :old_line) + end + + private + + def diff_lines + @diff_lines ||= @diff_file.diff_lines + end + + # Find old/new line number based on its old/new counterpart line number. + def map_line_number(from_line, from:, to:) + # If no diff file could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_file + + # To find the mapped line number for the specified line number, + # we need to find: + # - The diff line with that exact line number, if it is in the diff context + # - The first diff line with a higher line number, if it falls between diff contexts + # - The last known diff line, if it falls after the last diff context + diff_line = diff_lines.find do |diff_line| + diff_from_line = diff_line.send(from) + diff_from_line && diff_from_line >= from_line + end + diff_line ||= diff_lines.last + + # If no diff line could be found, the file wasn't changed, and the + # mapped line number is the same as the specified line number. + return from_line unless diff_line + + diff_from_line = diff_line.send(from) + diff_to_line = diff_line.send(to) + + # If the line was removed, there is no mapped line number. + return unless diff_to_line + + # Because we may not have the diff line with the exact line number + # we were looking for, we need to adjust the mapped line number. + distance = diff_from_line - from_line + + diff_to_line - distance + end + end + end +end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 74f9b3c050a..1c1fc148123 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -15,17 +15,19 @@ module Gitlab highlighted_diff_lines.each do |line| full_line = line.text type = line.type - line_code = generate_line_code(diff_file.file_path, line) + line_code = diff_file.line_code(line) line_new = line.new_pos line_old = line.old_pos + position = diff_file.position(line) next_line = diff_file.next_line(line.index) if next_line next_line = highlighted_diff_lines[next_line.index] - next_line_code = generate_line_code(diff_file.file_path, next_line) + full_next_line = next_line.text + next_line_code = diff_file.line_code(next_line) next_type = next_line.type - next_line = next_line.text + next_position = diff_file.position(next_line) end case type @@ -37,12 +39,14 @@ module Gitlab number: line_old, text: full_line, line_code: line_code, + position: position }, right: { type: type, number: line_new, text: full_line, - line_code: line_code + line_code: line_code, + position: position } } when 'old' @@ -55,12 +59,14 @@ module Gitlab number: line_old, text: full_line, line_code: line_code, + position: position }, right: { type: next_type, number: line_new, - text: next_line, - line_code: next_line_code + text: full_next_line, + line_code: next_line_code, + position: next_position, } } skip_next = true @@ -73,12 +79,14 @@ module Gitlab number: line_old, text: full_line, line_code: line_code, + position: position }, right: { type: next_type, number: nil, text: "", - line_code: nil + line_code: nil, + position: nil } } end @@ -95,12 +103,14 @@ module Gitlab number: nil, text: "", line_code: line_code, + position: position }, right: { type: type, number: line_new, text: full_line, - line_code: line_code + line_code: line_code, + position: position } } end @@ -108,12 +118,6 @@ module Gitlab end lines end - - private - - def generate_line_code(file_path, line) - Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) - end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 522dd2b9428..59a2367b65d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -40,7 +40,6 @@ module Gitlab line_obj_index += 1 end - case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb new file mode 100644 index 00000000000..989fff8918e --- /dev/null +++ b/lib/gitlab/diff/position.rb @@ -0,0 +1,155 @@ +# Defines a specific location, identified by paths and line numbers, +# within a specific diff, identified by start, head and base commit ids. +module Gitlab + module Diff + class Position + attr_reader :old_path + attr_reader :new_path + attr_reader :old_line + attr_reader :new_line + attr_reader :base_sha + attr_reader :start_sha + attr_reader :head_sha + + def initialize(attrs = {}) + @old_path = attrs[:old_path] + @new_path = attrs[:new_path] + @old_line = attrs[:old_line] + @new_line = attrs[:new_line] + + if attrs[:diff_refs] + @base_sha = attrs[:diff_refs].base_sha + @start_sha = attrs[:diff_refs].start_sha + @head_sha = attrs[:diff_refs].head_sha + else + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + end + end + + # `Gitlab::Diff::Position` objects are stored as serialized attributes in + # `DiffNote`, which use YAML to encode and decode objects. + # `#init_with` and `#encode_with` can be used to customize the en/decoding + # behavior. In this case, we override these to prevent memoized instance + # variables like `@diff_file` and `@diff_line` from being serialized. + def init_with(coder) + initialize(coder['attributes']) + + self + end + + def encode_with(coder) + coder['attributes'] = self.to_h + end + + def key + @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line] + end + + def ==(other) + other.is_a?(self.class) && key == other.key + end + + def to_h + { + old_path: old_path, + new_path: new_path, + old_line: old_line, + new_line: new_line, + base_sha: base_sha, + start_sha: start_sha, + head_sha: head_sha + } + end + + def inspect + %(#<#{self.class}:#{object_id} #{to_h}>) + end + + def complete? + file_path.present? && + (old_line || new_line) && + diff_refs.complete? + end + + def to_json + JSON.generate(self.to_h) + end + + def type + if old_line && new_line + nil + elsif new_line + 'new' + else + 'old' + end + end + + def unchanged? + type.nil? + end + + def added? + type == 'new' + end + + def removed? + type == 'old' + end + + def paths + [old_path, new_path].compact.uniq + end + + def file_path + new_path.presence || old_path + end + + def diff_refs + @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) + end + + def diff_file(repository) + @diff_file ||= begin + if RequestStore.active? + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + RequestStore.fetch(key) { find_diff_file(repository) } + else + find_diff_file(repository) + end + end + end + + def diff_line(repository) + @diff_line ||= diff_file(repository).line_for_position(self) + end + + def line_code(repository) + @line_code ||= diff_file(repository).line_code_for_position(self) + end + + private + + def find_diff_file(repository) + diffs = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ).diffs(paths: paths) + + diff = diffs.first + return unless diff + + Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb new file mode 100644 index 00000000000..4d04f867268 --- /dev/null +++ b/lib/gitlab/diff/position_tracer.rb @@ -0,0 +1,168 @@ +# Finds the diff position in the new diff that corresponds to the same location +# specified by the provided position in the old diff. +module Gitlab + module Diff + class PositionTracer + attr_accessor :repository + attr_accessor :old_diff_refs + attr_accessor :new_diff_refs + attr_accessor :paths + + def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) + @repository = repository + @old_diff_refs = old_diff_refs + @new_diff_refs = new_diff_refs + @paths = paths + end + + def trace(old_position) + return unless old_diff_refs.complete? && new_diff_refs.complete? + return unless old_position.diff_refs == old_diff_refs + + # Suppose we have an MR with source branch `feature` and target branch `master`. + # When the MR was created, the head of `master` was commit A, and the + # head of `feature` was commit B, resulting in the original diff A->B. + # Since creation, `master` was updated to C. + # Now `feature` is being updated to D, and the newly generated MR diff is C->D. + # It is possible that C and D are direct decendants of A and B respectively, + # but this isn't necessarily the case as rebases and merges come into play. + # + # Suppose we have a diff note on the original diff A->B. Now that the MR + # is updated, we need to find out what line in C->D corresponds to the + # line the note was originally created on, so that we can update the diff note's + # records and continue to display it in the right place in the diffs. + # If we cannot find this line in the new diff, this means the diff note is now + # outdated, and we will display that fact to the user. + # + # In the new diff, the file the diff note was originally created on may + # have been renamed, deleted or even created, if the file existed in A and B, + # but was removed in C, and restored in D. + # + # Every diff note stores a Position object that defines a specific location, + # identified by paths and line numbers, within a specific diff, identified + # by start, head and base commit ids. + # + # For diff notes for diff A->B, the position looks like this: + # Position + # base_sha - ID of commit A + # head_sha - ID of commit B + # old_path - path as of A (nil if file was newly created) + # new_path - path as of B (nil if file was deleted) + # old_line - line number as of A (nil if file was newly created) + # new_line - line number as of B (nil if file was deleted) + # + # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # but need to find the paths and line numbers as of C and D. + # + # If the file was unchanged or newly created in A->B, the path as of D can be found + # by generating diff B->D ("head to head"), finding the diff file with + # `diff_file.old_path == position.new_path`, and taking `diff_file.new_path`. + # The path as of C can be found by taking diff C->D, finding the diff file + # with that same `new_path` and taking `diff_file.old_path`. + # The line number as of D can be found by using the LineMapper on diff B->D + # and providing the line number as of B. + # The line number as of C can be found by using the LineMapper on diff C->D + # and providing the line number as of D. + # + # If the file was deleted in A->B, the path as of C can be found + # by generating diff A->C ("base to base"), finding the diff file with + # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. + # The path as of D can be found by taking diff C->D, finding the diff file + # with that same `old_path` and taking `diff_file.new_path`. + # The line number as of C can be found by using the LineMapper on diff A->C + # and providing the line number as of A. + # The line number as of D can be found by using the LineMapper on diff C->D + # and providing the line number as of C. + + results = nil + results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? + results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? + + return unless results + + file_diff, old_line, new_line = results + + Position.new( + old_path: file_diff.old_path, + new_path: file_diff.new_path, + head_sha: new_diff_refs.head_sha, + start_sha: new_diff_refs.start_sha, + base_sha: new_diff_refs.base_sha, + old_line: old_line, + new_line: new_line + ) + end + + private + + def trace_added_line(old_position) + file_path = old_position.new_path + + return unless diff_head_to_head + + file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_head_to_head.new_path if file_head_to_head + + new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) + + return unless new_line + + file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } + return unless file_diff + + old_line = LineMapper.new(file_diff).new_to_old(new_line) + + [file_diff, old_line, new_line] + end + + def trace_removed_line(old_position) + file_path = old_position.old_path + + return unless diff_base_to_base + + file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + + file_path = file_base_to_base.old_path if file_base_to_base + + old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + + return unless old_line + + file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } + return unless file_diff + + new_line = LineMapper.new(file_diff).old_to_new(old_line) + + [file_diff, old_line, new_line] + end + + def diff_base_to_base + @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + end + + def diff_head_to_head + @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + end + + def new_diffs + @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + end + + def diff_files(start_sha, head_sha, use_base: false) + base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + + diffs = self.repository.raw_repository.diff( + use_base ? base_sha : start_sha, + head_sha, + {}, + *paths + ) + + diffs.decorate! do |diff| + Gitlab::Diff::File.new(diff, repository: self.repository) + end + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 047c77c6fc2..97701b0cd42 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -33,11 +33,15 @@ module Gitlab end def commits - @commits ||= (Commit.decorate(compare.commits, project) if compare) + return unless compare + + @commits ||= Commit.decorate(compare.commits, project) end def diffs - @diffs ||= (safe_diff_files(compare.diffs(max_files: 30), diff_refs) if compare) + return unless compare + + @diffs ||= safe_diff_files(compare.diffs(max_files: 30), diff_refs: diff_refs, repository: project.repository) end def diffs_count diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 97ef9851d71..1c671a7487b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -104,15 +104,7 @@ module Gitlab end def create_note(reply) - Notes::CreateService.new( - sent_notification.project, - sent_notification.recipient, - note: reply, - noteable_type: sent_notification.noteable_type, - noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code - ).execute + sent_notification.create_note(reply) end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb new file mode 100644 index 00000000000..b63213ae208 --- /dev/null +++ b/lib/gitlab/emoji.rb @@ -0,0 +1,21 @@ +module Gitlab + module Emoji + extend self + + def emojis + Gemojione.index.instance_variable_get(:@emoji_by_name) + end + + def emojis_by_moji + Gemojione.index.instance_variable_get(:@emoji_by_moji) + end + + def emojis_names + emojis.keys.sort + end + + def emoji_filename(name) + emojis[name]["unicode"] + end + end +end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 07b856ca64c..9b681e636c7 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,6 +1,7 @@ module Gitlab module Git class Hook + GL_PROTOCOL = 'web'.freeze attr_reader :name, :repo_path, :path def initialize(name, repo_path) @@ -14,7 +15,7 @@ module Gitlab end def trigger(gl_id, oldrev, newrev, ref) - return true unless exists? + return [true, nil] unless exists? case name when "pre-receive", "post-receive" @@ -29,19 +30,20 @@ module Gitlab def call_receive_hook(gl_id, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") - # function will return true if succesful exit_status = false + exit_message = nil vars = { 'GL_ID' => gl_id, - 'PWD' => repo_path + 'PWD' => repo_path, + 'GL_PROTOCOL' => GL_PROTOCOL } options = { chdir: repo_path } - Open3.popen2(vars, path, options) do |stdin, _, wait_thr| + Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| exit_status = true stdin.sync = true @@ -60,17 +62,24 @@ module Gitlab unless wait_thr.value == 0 exit_status = false + exit_message = retrieve_error_message(stderr, stdout) end end - exit_status + [exit_status, exit_message] end def call_update_hook(gl_id, oldrev, newrev, ref) Dir.chdir(repo_path) do - system({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) + [status.success?, stderr.presence || stdout] end end + + def retrieve_error_message(stderr, stdout) + err_message = stderr.gets + err_message.blank? ? stdout.gets : err_message + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index d2a0e316cbe..7679c7e4bb8 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -3,11 +3,12 @@ module Gitlab DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } - attr_reader :actor, :project + attr_reader :actor, :project, :protocol - def initialize(actor, project) + def initialize(actor, project, protocol) @actor = actor @project = project + @protocol = protocol end def user @@ -49,6 +50,8 @@ module Gitlab end def check(cmd, changes = nil) + return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? + unless actor return build_status_object(false, "No user or key was provided.") end @@ -164,6 +167,10 @@ module Gitlab Gitlab::ForcePushCheck.force_push?(project, oldrev, newrev) end + def protocol_allowed? + Gitlab::ProtocolAccess.allowed?(protocol) + end + private def protected_branch_action(oldrev, newrev, branch_name) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index a15fc84b418..7d2d545b84e 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -4,7 +4,7 @@ module Gitlab delegate :repo, :sha, :ref, to: :raw_data def exists? - project.repository.branch_exists?(ref) + branch_exists? && commit_exists? end def name @@ -15,11 +15,15 @@ module Gitlab repo.present? end - def valid? - repo.present? + private + + def branch_exists? + project.repository.branch_exists?(ref) end - private + def commit_exists? + project.repository.commit(sha).present? + end def short_id sha.to_s[0..7] diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..043f10d96a9 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,7 +69,11 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..3932fcb1eda 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -131,8 +131,10 @@ module Gitlab def clean_up_restored_branches(branches) branches.each do |name, _| client.delete_ref(repo, "heads/#{name}") - project.repository.rm_branch(project.creator, name) + project.repository.delete_branch(name) rescue Rugged::ReferenceError end + + project.repository.after_remove_branch end def apply_labels(issuable) @@ -167,7 +169,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 498b00cb658..a4ea2210abd 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -11,10 +11,10 @@ module Gitlab description: description, source_project: source_branch_project, source_branch: source_branch_name, - head_source_sha: source_branch_sha, + source_branch_sha: source_branch_sha, target_project: target_branch_project, target_branch: target_branch_name, - base_target_sha: target_branch_sha, + target_branch_sha: target_branch_sha, state: state, milestone: milestone, author_id: author_id, diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f751a3a12fd..d4f12cb1df9 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,7 +3,6 @@ module Gitlab def add_gon_variables gon.api_version = API::API.version gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_shortcuts_path diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 2122339d2db..3caf9036459 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= (@commits.size.to_f / @duration).round(1) + @commit_per_day ||= @commits.size / (@duration + 1) end def collect_data diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 280120b0f9e..41296415e35 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,7 +1,7 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, nowrap: true, plain: false) - new(blob_name, blob_content, nowrap: nowrap). + def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false) + new(blob_name, blob_content, nowrap: nowrap, repository: repository). highlight(blob_content, continue: false, plain: plain) end @@ -10,12 +10,21 @@ module Gitlab return [] unless blob blob.load_all_data!(repository) - highlight(file_name, blob.data).lines.map!(&:html_safe) + highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end - def initialize(blob_name, blob_content, nowrap: true) + attr_reader :lexer + def initialize(blob_name, blob_content, repository: nil, nowrap: true) + @blob_name = blob_name + @blob_content = blob_content + @repository = repository @formatter = rouge_formatter(nowrap: nowrap) - @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText + + @lexer = custom_language || begin + Rouge::Lexer.guess(filename: blob_name, source: blob_content).new + rescue Rouge::Lexer::AmbiguousGuess => e + e.alternatives.sort_by(&:tag).first + end end def highlight(text, continue: true, plain: false) @@ -30,6 +39,14 @@ module Gitlab private + def custom_language + language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') + + return nil unless language_name + + Rouge::Lexer.find_fancy(language_name) + end + def rouge_formatter(options = {}) options = options.reverse_merge( nowrap: true, diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 99cf85d9a3b..588647e5adb 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.0' + VERSION = '0.1.1' def export_path(relative_path:) File.join(storage_path, relative_path) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 78664f076eb..2249904145c 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -28,7 +28,8 @@ module Gitlab end def execute(cmd) - _output, status = Gitlab::Popen.popen(cmd) + output, status = Gitlab::Popen.popen(cmd) + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status.zero? status.zero? end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 164ab6238c4..05f4ad527ac 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,24 +1,29 @@ # Model relationships to be included in the project import/export project_tree: - issues: + - :events - notes: - :author + - :author + - :events - :labels - - :milestones + - milestones: + - :events - snippets: - notes: :author - :releases - - :events - project_members: - :user - merge_requests: - notes: - :author + - :author + - :events - :merge_request_diff + - :events - pipelines: - notes: - :author + - :author + - :events - :statuses - :variables - :triggers diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 595b20a09bd..8f66f48cbfe 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Importer - def initialize(project) @archive_file = project.import_source @current_user = project.creator diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c569a35a48b..b459054c198 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids def initialize(exported_members:, user:, project:) diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index 89388d1984b..77bb3ca6581 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectCreator - def initialize(namespace_id, current_user, file, project_path) @namespace_id = namespace_id @current_user = current_user diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dd71b92c522..0ac6ff01e3b 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 19defd8f03a..15f5dd31035 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Reader - attr_reader :tree def initialize(shared:) @@ -55,7 +54,6 @@ module Gitlab @json_config_hash end - # If the model is a hash, process the sub_models, which could also be hashes # If there is a list, add to an existing array, otherwise use hash syntax # +current_key+ main model that will be a key in the hash diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 92bf7e0a2fc..9824df3f274 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class RelationFactory - OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', @@ -33,6 +32,7 @@ module Gitlab update_user_references update_project_references reset_ci_tokens if @relation_name == 'Ci::Trigger' + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] generate_imported_object end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index f38229c6c59..6a60b65071f 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -17,6 +17,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") archive_file else + @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) false end rescue => e diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 6aff05b886a..5d6de8bc475 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts def initialize(opts) diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 7292e9d9712..d6f4fa57510 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index cf5c62c5e3c..abfc694b879 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionChecker - def self.check!(*args) new(*args).check! end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index f7f73dc9343..9b642d740b7 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionSaver - def initialize(shared:) @shared = shared end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 948d43582cf..59a05411fe9 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -24,8 +24,6 @@ module Gitlab 'GitLab export' => 'gitlab_project' } end - end - end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index 8684b4636ea..b75ae512d92 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -39,7 +39,7 @@ module Gitlab # OpenSSH 6.8 introduces a new default output format for fingerprints. # Check the version and decide which command to use. - version_output, version_status = popen(%W(ssh -V)) + version_output, version_status = popen(%w(ssh -V)) return false unless version_status.zero? version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index e3ed2f6791d..811363405a8 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -1,7 +1,6 @@ module Gitlab module Lfs class Response - def initialize(project, user, ci, request) @origin_project = project @project = storage_project(project) diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index faf0d9b6318..c048fe20ba7 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -18,12 +18,12 @@ module Gitlab # Measures the real and CPU execution time of the supplied block. def measure - start_real = Time.now + start_real = System.monotonic_time start_cpu = System.cpu_time retval = yield - @real_time += (Time.now - start_real) * 1000.0 - @cpu_time += System.cpu_time.to_f - start_cpu + @real_time += System.monotonic_time - start_real + @cpu_time += System.cpu_time - start_cpu @call_count += 1 retval diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 1cd1ca30f70..f23d67e1e38 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,16 +4,15 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags, :created_at + attr_reader :series, :values, :tags # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. def initialize(series, values, tags = {}) - @values = values - @series = series - @tags = tags - @created_at = Time.now.utc + @values = values + @series = series + @tags = tags end # Returns a Hash in a format that can be directly written to InfluxDB. @@ -27,20 +26,20 @@ module Gitlab # # Due to the way InfluxDB is set up there's no solution to this problem, # all we can do is lower the amount of collisions. We do this by using - # Time#to_f which returns the seconds as a Float providing greater - # accuracy. We then add a small random value that is large enough to - # distinguish most timestamps but small enough to not alter the amount - # of seconds. + # System.real_time which returns the nanoseconds as a Float providing + # greater accuracy. We then add a small random value that is large + # enough to distinguish most timestamps but small enough to not alter + # the timestamp significantly. # # See https://gitlab.com/gitlab-com/operations/issues/175 for more # information. - time = @created_at.to_f + rand(JITTER_RANGE) + time = System.real_time(:nanosecond) + rand(JITTER_RANGE) { series: @series, tags: @tags, values: @values, - timestamp: (time * 1_000_000_000).to_i + timestamp: time.to_i } end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 8e345e8ae4a..aaed2184f44 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -2,11 +2,21 @@ module Gitlab module Metrics module Subscribers # Class for tracking the total time spent in Rails cache calls + # http://guides.rubyonrails.org/active_support_instrumentation.html class RailsCache < ActiveSupport::Subscriber attach_to :active_support def cache_read(event) increment(:cache_read, event.duration) + + return unless current_transaction + return if event.payload[:super_operation] == :fetch + + if event.payload[:hit] + current_transaction.increment(:cache_read_hit_count, 1) + else + current_transaction.increment(:cache_read_miss_count, 1) + end end def cache_write(event) @@ -21,6 +31,18 @@ module Gitlab increment(:cache_exists, event.duration) end + def cache_fetch_hit(event) + return unless current_transaction + + current_transaction.increment(:cache_read_hit_count, 1) + end + + def cache_generate(event) + return unless current_transaction + + current_transaction.increment(:cache_read_miss_count, 1) + end + def increment(key, duration) return unless current_transaction diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a7d183b2f94..82c18bb108b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,29 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f end else def self.cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f end end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float. + def self.real_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + end + + # Returns the current monotonic clock time in a given precision. + # + # Returns the time as a Float. + def self.monotonic_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 4bc5081aa03..bded245da43 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -30,7 +30,7 @@ module Gitlab end def duration - @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + @finished_at ? (@finished_at - @started_at) : 0.0 end def allocated_memory @@ -41,12 +41,12 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = Time.now + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = Time.now + @finished_at = System.monotonic_time Thread.current[THREAD_KEY] = nil end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 36e5c2670bb..7d6911a1ab3 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -66,7 +66,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7af75a9cc4c..0a91d3918d5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -56,8 +56,6 @@ module Gitlab if external_provider? && @user @user.external = true - elsif @user - @user.external = false end @user diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 746ec283330..4e2f8ed5587 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -1,7 +1,6 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup - # Public: Converts the provided markup into HTML. # # input - the source text in a markup format diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb new file mode 100644 index 00000000000..21aefc884be --- /dev/null +++ b/lib/gitlab/protocol_access.rb @@ -0,0 +1,13 @@ +module Gitlab + module ProtocolAccess + def self.allowed?(protocol) + if protocol == 'web' + true + elsif current_application_settings.enabled_git_access_protocol.blank? + true + else + protocol == current_application_settings.enabled_git_access_protocol + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c84c68f96f6..ffad5e17c78 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -13,7 +13,6 @@ module Gitlab "Cannot start with '-' or end in '.'." \ end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -22,7 +21,6 @@ module Gitlab "can contain only letters, digits, '_', '.', dash and space." end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -32,7 +30,6 @@ module Gitlab "It must start with letter, digit or '_'." end - def project_path_regex @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end @@ -42,7 +39,6 @@ module Gitlab "Cannot start with '-', end in '.git' or end in '.atom'" \ end - def file_name_regex @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end @@ -59,7 +55,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end - def directory_traversal_regex @directory_traversal_regex ||= /\.{2}/.freeze end @@ -68,7 +63,6 @@ module Gitlab "cannot include directory traversal. " end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 32c1c9ec5bb..67a5f368bdb 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash - def groups get_raw(Gitlab::Saml::Config.groups) end @@ -13,7 +12,6 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end - end end end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb index 0f40c00f547..574c3a4b28c 100644 --- a/lib/gitlab/saml/config.rb +++ b/lib/gitlab/saml/config.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class Config - class << self def options Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } @@ -15,7 +14,6 @@ module Gitlab options[:external_groups] end end - end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8943022612c..f253dc7477e 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -6,7 +6,6 @@ module Gitlab module Saml class User < Gitlab::OAuth::User - def save super('SAML') end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index ae85b294d31..104280f520a 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -25,18 +25,18 @@ module Gitlab Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ "#{MAX_RSS}" - Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\ + Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\ "in #{GRACE_TIME} seconds" sleep(GRACE_TIME) - Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill('SIGTERM', Process.pid) Sidekiq.logger.warn "waiting #{SHUTDOWN_WAIT} seconds before sending "\ - "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + "#{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" sleep(SHUTDOWN_WAIT) - Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}" + Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}" Process.kill(SHUTDOWN_SIGNAL, Process.pid) end end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb new file mode 100644 index 00000000000..b290c716f97 --- /dev/null +++ b/lib/gitlab/timeless.rb @@ -0,0 +1,16 @@ +module Gitlab + module Timeless + def self.timeless(model, &block) + original_record_timestamps = model.record_timestamps + model.record_timestamps = false + + if block.arity.abs == 1 + block.call(model) + else + block.call + end + ensure + model.record_timestamps = original_record_timestamps + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 7d02fe3c971..86ed18fb50d 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -6,8 +6,16 @@ module Gitlab content.gsub(regexp) { |url| new(url).masked_url } end + def self.valid?(url) + Addressable::URI.parse(url.strip) + + true + rescue Addressable::URI::InvalidURIError + false + end + def initialize(url, credentials: nil) - @url = Addressable::URI.parse(url) + @url = Addressable::URI.parse(url.strip) @credentials = credentials end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 40e8299c36b..bc0193a6c32 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -38,12 +38,10 @@ module Gitlab end def send_git_diff(repository, diff_refs) - from, to = diff_refs - params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => from.sha, - 'ShaTo' => to.sha + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha } [ @@ -52,6 +50,19 @@ module Gitlab ] end + def send_git_patch(repository, diff_refs) + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => diff_refs.start_sha, + 'ShaTo' => diff_refs.head_sha + } + + [ + SEND_DATA_HEADER, + "git-format-patch:#{encode(params)}" + ] + end + protected def encode(hash) |