diff options
Diffstat (limited to 'lib/gitlab')
233 files changed, 4677 insertions, 2060 deletions
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index e4f7cad2b79..45c2b01dd8f 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -1,7 +1,7 @@ module Gitlab module Allowable - def can?(user, action, subject = :global) - Ability.allowed?(user, action, subject) + def can?(*args) + Ability.allowed?(*args) end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 099c45dcfb7..9bed81e7327 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,8 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) + REGISTRY_SCOPES = [:read_registry].freeze + # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze @@ -11,8 +13,10 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze + AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze + # Other available scopes - OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze + OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) @@ -26,20 +30,28 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password) || oauth_access_token_check(login, password) || - user_with_password_for_git(login, password) || personal_access_token_check(password) || + user_with_password_for_git(login, password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - result + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + + # If sign-in is disabled and LDAP is not configured, recommend a + # personal access token on failed auth attempts + raise Gitlab::Auth::MissingPersonalTokenError end def find_with_user_password(login, password) # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -109,6 +121,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) @@ -121,17 +134,23 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_api_token?(token) - Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_api_token?(token) + token && token.accessible? && valid_scoped_token?(token, [:api]) end - def valid_api_token?(token) - AccessTokenValidationService.new(token).include_any_scope?(['api']) + def valid_scoped_token?(token, scopes) + AccessTokenValidationService.new(token).include_any_scope?(scopes) + end + + def abilities_for_scope(scopes) + scopes.map do |scope| + self.public_send(:"#{scope}_scope_authentication_abilities") + end.flatten.uniq end def lfs_token_check(login, password) @@ -202,6 +221,16 @@ module Gitlab :create_container_image ] end + alias_method :api_scope_authentication_abilities, :full_authentication_abilities + + def read_registry_scope_authentication_abilities + [:read_container_image] + end + + # The currently used auth method doesn't allow any actions for this scope + def read_user_scope_authentication_abilities + [] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 39b86c61a18..75451cf8aa9 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -15,6 +15,10 @@ module Gitlab def success? actor.present? || type == :ci end + + def failed? + !success? + end end end end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb new file mode 100644 index 00000000000..d3f66877672 --- /dev/null +++ b/lib/gitlab/background_migration.rb @@ -0,0 +1,52 @@ +module Gitlab + module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + + # Begins stealing jobs from the background migrations queue, blocking the + # caller until all jobs have been completed. + # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # + # steal_class - The name of the class for which to steal jobs. + def self.steal(steal_class) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new + + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args + + next unless job.queue == self.queue + next unless migration_class == steal_class + + begin + perform(migration_class, migration_args) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) + + raise + end + end + end + end + + ## + # Performs a background migration. + # + # class_name - The name of the background migration class as defined in the + # Gitlab::BackgroundMigration namespace. + # + # arguments - The arguments to pass to the background migration's "perform" + # method. + def self.perform(class_name, arguments) + const_get(class_name).new.perform(*arguments) + end + end +end diff --git a/lib/gitlab/background_migration/.gitkeep b/lib/gitlab/background_migration/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/lib/gitlab/background_migration/.gitkeep diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb new file mode 100644 index 00000000000..91540127ea9 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -0,0 +1,19 @@ +module Gitlab + module BackgroundMigration + class MigrateBuildStageIdReference + def perform(start_id, stop_id) + sql = <<-SQL.strip_heredoc + UPDATE ci_builds + SET stage_id = + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND ci_builds.stage_id IS NULL + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb new file mode 100644 index 00000000000..0881244ed49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -0,0 +1,26 @@ +module Gitlab + module BackgroundMigration + class MigrateSystemUploadsToNewFolder + include Gitlab::Database::MigrationHelpers + attr_reader :old_folder, :new_folder + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + include EachBatch + end + + def perform(old_folder, new_folder) + replace_sql = replace_sql(uploads[:path], old_folder, new_folder) + affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) + + affected_uploads.each_batch do |batch| + batch.update_all("path = #{replace_sql}") + end + end + + def uploads + Arel::Table.new('uploads') + end + end + end +end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb index 53588185622..e898f5d790e 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -16,13 +16,11 @@ module Gitlab end def image_url - coverage_namespace_project_badges_url(@project.namespace, - @project, @ref, - format: :svg) + coverage_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, @ref) end end end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 4a049ef758d..8ad6f3cb986 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -4,7 +4,7 @@ module Gitlab # Abstract class for badge metadata # class Metadata - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing include ActionView::Helpers::AssetTagHelper include ActionView::Helpers::UrlHelper diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/pipeline/metadata.rb index f87a7b7942e..db1e9f8cfb8 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/pipeline/metadata.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that describes build badge metadata + # Class that describes pipeline badge metadata # class Metadata < Badge::Metadata def initialize(badge) @@ -11,16 +11,15 @@ module Gitlab end def title - 'build status' + 'pipeline status' end def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) + pipeline_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, id: @ref) end end end diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/pipeline/status.rb index b762d85b6e5..5fee7a93475 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Build status badge + # Pipeline status badge # class Status < Badge::Base attr_reader :project, :ref @@ -15,7 +15,7 @@ module Gitlab end def entity - 'build' + 'pipeline' end def status @@ -25,11 +25,11 @@ module Gitlab end def metadata - @metadata ||= Build::Metadata.new(self) + @metadata ||= Pipeline::Metadata.new(self) end def template - @template ||= Build::Template.new(self) + @template ||= Pipeline::Template.new(self) end end end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/pipeline/template.rb index bc0e0cd441d..e09db32262d 100644 --- a/lib/gitlab/badge/build/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -1,12 +1,13 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that represents a build badge template. + # Class that represents a pipeline badge template. # # Template object will be passed to badge.svg.erb template. # class Template < Badge::Template + STATUS_RENAME = { 'success' => 'passed' }.freeze STATUS_COLOR = { success: '#4c1', failed: '#e05d44', @@ -27,11 +28,11 @@ module Gitlab end def value_text - @status.to_s + STATUS_RENAME[@status.to_s] || @status.to_s end def key_width - 38 + 62 end def value_width diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 5a6d9ae99a0..28bbf3b384e 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -61,7 +61,7 @@ module Gitlab def import_wiki return if project.wiki.repository_exists? - path_with_namespace = "#{project.path_with_namespace}.wiki" + path_with_namespace = "#{project.full_path}.wiki" import_url = project.import_url.sub(/\.git\z/, ".git/wiki") gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) rescue StandardError => e diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index d62bc50ce78..169aac79854 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -40,7 +40,7 @@ module Gitlab end def highlighted_lines - @blob.load_all_data!(repository) + @blob.load_all_data! @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 4fc9a075edc..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -50,8 +50,8 @@ module Gitlab ref: pipeline.ref } - new(pipeline.project, pipeline_info: pipeline_info). - store_in_cache_if_needed + new(pipeline.project, pipeline_info: pipeline_info) + .store_in_cache_if_needed end def initialize(project, pipeline_info: {}, loaded_from_cache: nil) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb new file mode 100644 index 00000000000..f1a04affd38 --- /dev/null +++ b/lib/gitlab/cache/request_cache.rb @@ -0,0 +1,94 @@ +module Gitlab + module Cache + # This module provides a simple way to cache values in RequestStore, + # and the cache key would be based on the class name, method name, + # optionally customized instance level values, optionally customized + # method level values, and optional method arguments. + # + # A simple example: + # + # class UserAccess + # extend Gitlab::Cache::RequestCache + # + # request_cache_key do + # [user&.id, project&.id] + # end + # + # request_cache def can_push_to_branch?(ref) + # # ... + # end + # end + # + # This way, the result of `can_push_to_branch?` would be cached in + # `RequestStore.store` based on the cache key. If RequestStore is not + # currently active, then it would be stored in a hash saved in an + # instance variable, so the cache logic would be the same. + # Here's another example using customized method level values: + # + # class Commit + # extend Gitlab::Cache::RequestCache + # + # def author + # User.find_by_any_email(author_email.downcase) + # end + # request_cache(:author) { author_email.downcase } + # end + # + # So that we could have different strategies for different methods + # + module RequestCache + def self.extended(klass) + return if klass < self + + extension = Module.new + klass.const_set(:RequestCacheExtension, extension) + klass.prepend(extension) + end + + def request_cache_key(&block) + if block_given? + @request_cache_key = block + else + @request_cache_key + end + end + + def request_cache(method_name, &method_key_block) + const_get(:RequestCacheExtension).module_eval do + cache_key_method_name = "#{method_name}_cache_key" + + define_method(method_name) do |*args| + store = + if RequestStore.active? + RequestStore.store + else + ivar_name = # ! and ? cannot be used as ivar name + "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" + + instance_variable_get(ivar_name) || + instance_variable_set(ivar_name, {}) + end + + key = __send__(cache_key_method_name, args) + + store.fetch(key) { store[key] = super(*args) } + end + + define_method(cache_key_method_name) do |args| + klass = self.class + + instance_key = instance_exec(&klass.request_cache_key) if + klass.request_cache_key + + method_key = instance_exec(&method_key_block) if method_key_block + + [klass.name, method_name, *instance_key, *method_key, *args] + .join(':') + end + + private cache_key_method_name + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/deploy.rb b/lib/gitlab/chat_commands/presenters/deploy.rb deleted file mode 100644 index 863d0bf99ca..00000000000 --- a/lib/gitlab/chat_commands/presenters/deploy.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Gitlab - module ChatCommands - module Presenters - class Deploy < Presenters::Base - def present(from, to) - message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." - - in_channel_response(text: message) - end - - def no_actions - ephemeral_response(text: "No action found to be executed") - end - - def too_many_actions - ephemeral_response(text: "Too many actions defined") - end - end - end - end -end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index c62aeb60fa9..b88b2e36d53 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -2,7 +2,7 @@ module Gitlab module Ci module Build class Image - attr_reader :name + attr_reader :alias, :command, :entrypoint, :name class << self def from_image(job) @@ -21,7 +21,14 @@ module Gitlab end def initialize(image) - @name = image + if image.is_a?(String) + @name = image + elsif image.is_a?(Hash) + @alias = image[:alias] + @command = image[:command] + @entrypoint = image[:entrypoint] + @name = image[:name] + end end def valid? diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f074df9c7a1..d7e09acbbf3 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,11 +7,14 @@ module Gitlab # class Cache < Node include Configurable + include Attributable - ALLOWED_KEYS = %i[key untracked paths].freeze + ALLOWED_KEYS = %i[key untracked paths policy].freeze + DEFAULT_POLICY = 'pull-push'.freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true end entry :key, Entry::Key, @@ -25,8 +28,15 @@ module Gitlab helpers :key + attributes :policy + def value - super.merge(key: key_value) + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + + result end end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index b5050257688..6555c589173 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -8,8 +8,36 @@ module Gitlab class Image < Node include Validatable + ALLOWED_KEYS = %i[name entrypoint].freeze + validations do - validates :config, type: String + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :entrypoint, array_of_strings: true, allow_nil: true + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def entrypoint + value[:entrypoint] + end + + def value + return { name: @config } if string? + return @config if hash? + {} end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 176301bcca1..32f5c6ab142 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment coverage].freeze + after_script variables environment coverage retry].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -23,6 +23,9 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true + validates :retry, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ @@ -76,9 +79,9 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage + :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies + attributes :script, :tags, :allow_failure, :when, :dependencies, :retry def compose!(deps = nil) super do @@ -142,6 +145,7 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, + retry: retry_defined? ? retry_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb new file mode 100644 index 00000000000..3e2ebcff31a --- /dev/null +++ b/lib/gitlab/ci/config/entry/service.rb @@ -0,0 +1,34 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of Docker service. + # + class Service < Image + include Validatable + + ALLOWED_KEYS = %i[name entrypoint command alias].freeze + + validations do + validates :config, hash_or_string: true + validates :config, allowed_keys: ALLOWED_KEYS + + validates :name, type: String, presence: true + validates :entrypoint, array_of_strings: true, allow_nil: true + validates :command, array_of_strings: true, allow_nil: true + validates :alias, type: String, allow_nil: true + end + + def alias + value[:alias] + end + + def command + value[:command] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 84f8ab780f5..0066894e069 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -9,7 +9,30 @@ module Gitlab include Validatable validations do - validates :config, array_of_strings: true + validates :config, type: Array + end + + def compose!(deps = nil) + super do + @entries = [] + @config.each do |config| + @entries << Entry::Factory.new(Entry::Service) + .value(config || {}) + .create! + end + + @entries.each do |entry| + entry.compose!(deps) + end + end + end + + def value + @entries.map(&:value) + end + + def descendants + @entries end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index bd7428b1272..b2ca3c881e4 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -44,6 +44,14 @@ module Gitlab end end + class HashOrStringValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(String) + record.errors.add(attribute, 'should be a hash or a string') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb index a210e76acaa..3208cc2bef6 100644 --- a/lib/gitlab/ci/pipeline_duration.rb +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -87,8 +87,8 @@ module Gitlab def from_pipeline(pipeline) status = %w[success failed running canceled] - builds = pipeline.builds.latest. - where(status: status).where.not(started_at: nil).order(:started_at) + builds = pipeline.builds.latest + .where(status: status).where.not(started_at: nil).order(:started_at) from_builds(builds) end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb new file mode 100644 index 00000000000..f81f9347b4d --- /dev/null +++ b/lib/gitlab/ci/stage/seed.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + module Stage + class Seed + attr_reader :pipeline + delegate :project, to: :pipeline + + def initialize(pipeline, stage, jobs) + @pipeline = pipeline + @stage = { name: stage } + @jobs = jobs.to_a.dup + end + + def user=(current_user) + @jobs.map! do |attributes| + attributes.merge(user: current_user) + end + end + + def stage + @stage.merge(project: project) + end + + def builds + trigger = pipeline.trigger_requests.first + + @jobs.map do |attributes| + attributes.merge(project: project, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: trigger) + end + end + + def create! + pipeline.stages.create!(stage).tap do |stage| + builds_attributes = builds.map do |attributes| + attributes.merge(stage_id: stage.id) + end + + pipeline.builds.create!(builds_attributes).each do |build| + yield build if block_given? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 439ef0ce015..8ad3e57e59d 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,9 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + cancel_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index b173c23fba4..c0c7c7f5b5d 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + project_job_path(subject.project, subject) end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index e80f3263794..c7726543599 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 56303e4cb17..8c8fdc56d75 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,9 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + retry_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 2778d6f3b52..d464738deaf 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index 97c121ce7b9..e5fdc1f8136 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Canceled < Status::Core def text - 'canceled' + s_('CiStatusText|canceled') end def label - 'canceled' + s_('CiStatusLabel|canceled') end def icon diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 0721bf6ec7c..d188bd286a6 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Created < Status::Core def text - 'created' + s_('CiStatusText|created') end def label - 'created' + s_('CiStatusLabel|created') end def icon diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 4969a350862..9307545b5b1 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -3,6 +3,10 @@ module Gitlab module Status module External module Common + def label + subject.description + end + def has_details? subject.target_url.present? && can?(user, :read_commit_status, subject) diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index cb75e9383a8..38e45714c22 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Failed < Status::Core def text - 'failed' + s_('CiStatusText|failed') end def label - 'failed' + s_('CiStatusLabel|failed') end def icon diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index f8f6c2903ba..a4a7edadac9 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Manual < Status::Core def text - 'manual' + s_('CiStatusText|manual') end def label - 'manual action' + s_('CiStatusLabel|manual action') end def icon diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index f40cc1314dc..5164260b861 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Pending < Status::Core def text - 'pending' + s_('CiStatusText|pending') end def label - 'pending' + s_('CiStatusLabel|pending') end def icon diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index 37dfe43fb62..bf7e484ee9b 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -4,11 +4,11 @@ module Gitlab module Pipeline class Blocked < Status::Extended def text - 'blocked' + s_('CiStatusText|blocked') end def label - 'waiting for manual action' + s_('CiStatusLabel|waiting for manual action') end def self.matches?(pipeline, user) diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 76bfd18bf40..61bb07beb0f 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject) + project_pipeline_path(subject.project, subject) end def has_action? diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index 1237cd47dc8..993937e98ca 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Running < Status::Core def text - 'running' + s_('CiStatus|running') end def label - 'running' + s_('CiStatus|running') end def icon diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 28005d91503..0c942920b02 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Skipped < Status::Core def text - 'skipped' + s_('CiStatusText|skipped') end def label - 'skipped' + s_('CiStatusLabel|skipped') end def icon diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index 7852f492e1d..bc99d925347 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -8,10 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject.pipeline, - anchor: subject.name) + project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) end def has_action? diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 88f7758a270..d7af98857b0 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Success < Status::Core def text - 'passed' + s_('CiStatusText|passed') end def label - 'passed' + s_('CiStatusLabel|passed') end def icon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index df6e76b0151..4d7d82e04cf 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -7,11 +7,11 @@ module Gitlab # class SuccessWarning < Status::Extended def text - 'passed' + s_('CiStatusText|passed') end def label - 'passed with warnings' + s_('CiStatusLabel|passed with warnings') end def icon diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c4c0623df6c..8503ecf8700 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -67,14 +67,15 @@ module Gitlab def extract_coverage(regex) return unless valid? - return unless regex + return unless regex.present? - regex = Regexp.new(regex) + regex = Gitlab::UntrustedRegexp.new(regex) match = "" reverse_line do |line| - matches = line.scan(regex) + line.chomp! + matches = regex.scan(line) next unless matches.is_a?(Array) next if matches.empty? diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 75a213ef752..98dfe900044 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,7 +1,7 @@ module Gitlab module Conflict class File - include Gitlab::Routing.url_helpers + include Gitlab::Routing include IconsHelper MissingResolution = Class.new(ResolutionError) @@ -205,9 +205,7 @@ module Gitlab old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), - blob_path: namespace_project_blob_path(merge_request.project.namespace, - merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)) + blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path)) } json_hash.tap do |json_hash| @@ -223,11 +221,10 @@ module Gitlab end def content_path - conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, - merge_request.project, - merge_request, - old_path: their_path, - new_path: our_path) + conflict_for_path_project_merge_request_path(merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 6e73361cad1..1611eba31da 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -16,9 +16,9 @@ module Gitlab project = merge_request.source_project new(merge_request, project).tap do |file_collection| - project. - repository. - with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + project + .repository + .with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do yield file_collection end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 15992b77680..bf557103cfd 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -16,27 +16,27 @@ module Gitlab # Can't use Event.contributions here because we need to check 3 different # project_features for the (currently) 3 different contribution types date_from = 1.year.ago - repo_events = event_counts(date_from, :repository). - having(action: Event::PUSHED) - issue_events = event_counts(date_from, :issues). - having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") - mr_events = event_counts(date_from, :merge_requests). - having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") - note_events = event_counts(date_from, :merge_requests). - having(action: [Event::COMMENTED], target_type: "Note") + repo_events = event_counts(date_from, :repository) + .having(action: Event::PUSHED) + issue_events = event_counts(date_from, :issues) + .having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + mr_events = event_counts(date_from, :merge_requests) + .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") + note_events = event_counts(date_from, :merge_requests) + .having(action: [Event::COMMENTED], target_type: "Note") union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) - @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end def events_by_date(date) - events = Event.contributions.where(author_id: contributor.id). - where(created_at: date.beginning_of_day..date.end_of_day). - where(project_id: projects) + events = Event.contributions.where(author_id: contributor.id) + .where(created_at: date.beginning_of_day..date.end_of_day) + .where(project_id: projects) # Use visible_to_user? instead of the complicated logic in activity_dates # because we're only viewing the events for a single day. @@ -60,20 +60,20 @@ module Gitlab # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.uniq.pluck(:id) - authed_projects = Project.where(id: @contributed_project_ids). - with_feature_available_for_user(feature, current_user). - reorder(nil). - select(:id) + authed_projects = Project.where(id: @contributed_project_ids) + .with_feature_available_for_user(feature, current_user) + .reorder(nil) + .select(:id) - conditions = t[:created_at].gteq(date_from.beginning_of_day). - and(t[:created_at].lteq(Date.today.end_of_day)). - and(t[:author_id].eq(contributor.id)) + conditions = t[:created_at].gteq(date_from.beginning_of_day) + .and(t[:created_at].lteq(Date.today.end_of_day)) + .and(t[:author_id].eq(contributor.id)) - Event.reorder(nil). - select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount'). - group(t[:project_id], t[:target_type], t[:action], 'date(created_at)'). - where(conditions). - having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + Event.reorder(nil) + .select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount') + .group(t[:project_id], t[:target_type], t[:action], 'date(created_at)') + .where(conditions) + .having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 48735fd197d..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -10,43 +10,44 @@ module Gitlab delegate :sidekiq_throttling_enabled?, to: :current_application_settings - def fake_application_settings - OpenStruct.new(::ApplicationSetting.defaults) + def fake_application_settings(defaults = ::ApplicationSetting.defaults) + FakeApplicationSettings.new(defaults) end private def ensure_application_settings! - unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - settings = retrieve_settings_from_database? - end + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - settings || in_memory_application_settings + cached_application_settings || uncached_application_settings end - def retrieve_settings_from_database? - settings = retrieve_settings_from_database_cache? - return settings if settings.present? - - return fake_application_settings unless connect_to_db? - + def cached_application_settings begin - db_settings = ::ApplicationSetting.current + ::ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Redis::BaseError, ::Errno::ENOENT - db_settings = ::ApplicationSetting.last end - db_settings || ::ApplicationSetting.create_from_defaults end - def retrieve_settings_from_database_cache? - begin - settings = ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - settings = nil + def uncached_application_settings + return fake_application_settings unless connect_to_db? + + db_settings = ::ApplicationSetting.current + + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ::ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) end - settings + + return db_settings if db_settings.present? + + ::ApplicationSetting.create_from_defaults || in_memory_application_settings end def in_memory_application_settings @@ -62,8 +63,7 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') && - !ActiveRecord::Migrator.needs_migration? + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index d560dca45c8..58729d3ced8 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -12,17 +12,17 @@ module Gitlab end def stage_query - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). - join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). - where(issue_table[:project_id].eq(@project.id)). - where(issue_table[:deleted_at].eq(nil)). - where(issue_table[:created_at].gteq(@options[:from])) + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) + .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .where(issue_table[:project_id].eq(@project.id)) + .where(issue_table[:deleted_at].eq(nil)) + .where(issue_table[:created_at].gteq(@options[:from])) # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin). - on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). - join(mr_metrics_table). - on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + query = query.join(mr_table, Arel::Nodes::OuterJoin) + .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) + .join(mr_metrics_table) + .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) query end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 9d25ef078e8..f5d08c0b658 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -13,6 +13,10 @@ module Gitlab MergeRequestDiff.arel_table end + def mr_diff_commits_table + MergeRequestDiffCommit.arel_table + end + def mr_closing_issues_table MergeRequestsClosingIssues.arel_table end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7d342a2d2cb..b260822788d 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -2,40 +2,59 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @projections = [mr_diff_table[:st_commits].as('commits'), + @projections = [mr_diff_table[:id], + mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end def events_query - base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + base_query + .join(mr_diff_table) + .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) super end private + def merge_request_diff_commits + @merge_request_diff_commits ||= + MergeRequestDiffCommit + .where(merge_request_diff_id: event_result.map { |event| event['id'] }) + .group_by(&:merge_request_diff_id) + end + def serialize(event) - st_commit = first_time_reference_commit(event.delete('commits'), event) + commit = first_time_reference_commit(event) - return unless st_commit + return unless commit - serialize_commit(event, st_commit, query) + serialize_commit(event, commit, query) end - def first_time_reference_commit(commits, event) + def first_time_reference_commit(event) + return nil unless event && merge_request_diff_commits + + commits = + if event['st_commits'].present? + YAML.load(event['st_commits']) + else + merge_request_diff_commits[event['id'].to_i] + end + return nil if commits.blank? - YAML.load(commits).find do |commit| + commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i end end - def serialize_commit(event, st_commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + def serialize_commit(event, commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 182a30fd74d..e47fb85b5ee 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages_name, + stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index e81d19a7a2e..5c5f507d44d 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -24,11 +24,11 @@ module Gitlab # total_commits_count: Fixnum # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) commits = Array(commits) # Total commits count - commits_count = commits.size + commits_count ||= commits.size # Get latest 20 commits ASC commits_limited = commits.last(20) @@ -74,6 +74,8 @@ module Gitlab build(project, user, commits.last&.id, commits.first&.id, ref, commits) end + private + def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb new file mode 100644 index 00000000000..226974b698c --- /dev/null +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -0,0 +1,22 @@ +module Gitlab + module DataBuilder + module WikiPage + extend self + + def build(wiki_page, user, action) + wiki = wiki_page.wiki + + { + object_kind: wiki_page.class.name.underscore, + user: user.hook_attrs, + project: wiki.project.hook_attrs, + wiki: wiki.hook_attrs, + object_attributes: wiki_page.hook_attrs.merge( + url: Gitlab::UrlBuilder.build(wiki_page), + action: action + ) + } + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d0bd1299671..d7dab584a44 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -83,6 +83,22 @@ module Gitlab end end + def self.bulk_insert(table, rows) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + tuples = rows.map do |row| + row.values_at(*keys).map { |value| connection.quote(value) } + end + + connection.execute <<-EOF + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + end + # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil) diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 23890e5f493..059054ac9ff 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -29,10 +29,10 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table. - from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). - project(average([arel_table[column_sym]], 'median')). - where( + query = arel_table + .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) + .project(average([arel_table[column_sym]], 'median')) + .where( Arel::Nodes::Between.new( Arel.sql("(select @row_id := @row_id + 1)"), Arel::Nodes::And.new( @@ -67,8 +67,8 @@ module Gitlab cte_table = Arel::Table.new("ordered_records") cte = Arel::Nodes::As.new( cte_table, - arel_table. - project( + arel_table + .project( arel_table[column_sym].as(column_sym.to_s), Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), @@ -79,8 +79,8 @@ module Gitlab # From the CTE, select either the middle row or the middle two rows (this is accomplished # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the # selected rows, and this is the median value. - cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). - where( + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")) + .where( Arel::Nodes::Between.new( cte_table[:row_id], Arel::Nodes::And.new( @@ -88,9 +88,9 @@ module Gitlab (cte_table[:ct] / Arel.sql('2.0') + 1)] ) ) - ). - with(query_so_far, cte). - to_sql + ) + .with(query_so_far, cte) + .to_sql end private diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a412bb6dbd2..69ca9aa596b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,6 +1,39 @@ module Gitlab module Database module MigrationHelpers + # Adds `created_at` and `updated_at` columns with timezone information. + # + # This method is an improved version of Rails' built-in method `add_timestamps`. + # + # Available options are: + # default - The default value for the column. + # null - When set to `true` the column will allow NULL values. + # The default is to not allow NULL values. + def add_timestamps_with_timezone(table_name, options = {}) + options[:null] = false if options[:null].nil? + + [:created_at, :updated_at].each do |column_name| + if options[:default] && transaction_open? + raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` ' \ + 'in the body of your migration class' + end + + # If default value is presented, use `add_column_with_default` method instead. + if options[:default] + add_column_with_default( + table_name, + column_name, + :datetime_with_timezone, + default: options[:default], + allow_null: options[:null] + ) + else + add_column(table_name, column_name, :datetime_with_timezone, options) + end + end + end + # Creates a new index, concurrently when supported # # On PostgreSQL this method creates an index concurrently, on MySQL this @@ -107,6 +140,8 @@ module Gitlab return add_foreign_key(source, target, column: column, on_delete: on_delete) + else + on_delete = 'SET NULL' if on_delete == :nullify end disable_statement_timeout @@ -122,7 +157,7 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete}" : ''} + #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} NOT VALID; EOF @@ -189,6 +224,12 @@ module Gitlab # # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) + if transaction_open? + raise 'update_column_in_batches can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + table = Arel::Table.new(table) count_arel = table.project(Arel.star.count.as('count')) @@ -200,25 +241,31 @@ module Gitlab # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil + max_size = 1000 + + # The upper limit is 1000 to ensure we don't lock too many rows. For + # example, for "merge_requests" even 1% of the table is around 35 000 + # rows for GitLab.com. + batch_size = max_size if batch_size > max_size start_arel = table.project(table[:id]).order(table[:id].asc).take(1) start_arel = yield table, start_arel if block_given? start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i loop do - stop_arel = table.project(table[:id]). - where(table[:id].gteq(start_id)). - order(table[:id].asc). - take(1). - skip(batch_size) + stop_arel = table.project(table[:id]) + .where(table[:id].gteq(start_id)) + .order(table[:id].asc) + .take(1) + .skip(batch_size) stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = Arel::UpdateManager.new(ActiveRecord::Base). - table(table). - set([[table[column], value]]). - where(table[:id].gteq(start_id)) + update_arel = Arel::UpdateManager.new(ActiveRecord::Base) + .table(table) + .set([[table[column], value]]) + .where(table[:id].gteq(start_id)) if stop_row stop_id = stop_row['id'].to_i @@ -547,15 +594,15 @@ module Gitlab quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) if Database.mysql? - locate = Arel::Nodes::NamedFunction. - new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction. - new('insert', [column, locate, pattern.size, quoted_replacement]) + locate = Arel::Nodes::NamedFunction + .new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction + .new('insert', [column, locate, pattern.size, quoted_replacement]) Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) else - replace = Arel::Nodes::NamedFunction. - new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + replace = Arel::Nodes::NamedFunction + .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) Arel::Nodes::SqlLiteral.new(replace.to_sql) end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb index 89530082cd2..f333ff22300 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -29,6 +29,11 @@ module Gitlab paths = Array(paths) RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) end + + def revert_renames + RenameProjects.new([], self).revert_renames + RenameNamespaces.new([], self).revert_renames + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index d60fd4bb551..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -6,7 +6,10 @@ module Gitlab attr_reader :paths, :migration delegate :update_column_in_batches, + :execute, :replace_sql, + :quote_string, + :say, to: :migration def initialize(paths, migration) @@ -26,24 +29,45 @@ module Gitlab new_path = rename_path(namespace_path, old_path) new_full_path = join_routable_path(namespace_path, new_path) + perform_rename(routable, old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def perform_rename(routable, old_full_path, new_full_path) # skips callbacks & validations - routable.class.where(id: routable). - update_all(path: new_path) + new_path = new_full_path.split('/').last + routable.class.where(id: routable) + .update_all(path: new_path) rename_routes(old_full_path, new_full_path) - - [old_full_path, new_full_path] end def rename_routes(old_full_path, new_full_path) + routes = Route.arel_table + + quoted_old_full_path = quote_string(old_full_path) + quoted_old_wildcard_path = quote_string("#{old_full_path}/%") + + filter = if Database.mysql? + "lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "OR routes.path LIKE '#{quoted_old_wildcard_path}'" + else + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" + end + replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path) - update_column_in_batches(:routes, :path, replace_statement) do |table, query| - path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) - query.where(path_or_children) - end + update = Arel::UpdateManager.new(ActiveRecord::Base) + .table(routes) + .set([[routes[:path], replace_statement]]) + .where(Arel::Nodes::SqlLiteral.new(filter)) + + execute(update.to_sql) end def rename_path(namespace_path, path_was) @@ -86,32 +110,74 @@ module Gitlab def move_folders(directory, old_relative_path, new_relative_path) old_path = File.join(directory, old_relative_path) - return unless File.directory?(old_path) + unless File.directory?(old_path) + say "#{old_path} doesn't exist, skipping" + return + end new_path = File.join(directory, new_relative_path) FileUtils.mv(old_path, new_path) end def remove_cached_html_for_projects(project_ids) - update_column_in_batches(:projects, :description_html, nil) do |table, query| - query.where(table[:id].in(project_ids)) - end - - update_column_in_batches(:issues, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) + project_ids.each do |project_id| + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].eq(project_id)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].eq(project_id)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end end + end - update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| - query.where(table[:target_project_id].in(project_ids)) + def track_rename(type, old_path, new_path) + key = redis_key_for_type(type) + Gitlab::Redis::SharedState.with do |redis| + redis.lpush(key, [old_path, new_path].to_json) + redis.expire(key, 2.weeks.to_i) end + say "tracked rename: #{key}: #{old_path} -> #{new_path}" + end - update_column_in_batches(:notes, :note_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) + def reverts_for_type(type) + key = redis_key_for_type(type) + + Gitlab::Redis::SharedState.with do |redis| + failed_reverts = [] + + while rename_info = redis.lpop(key) + path_before_rename, path_after_rename = JSON.parse(rename_info) + say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" + begin + yield(path_before_rename, path_after_rename) + rescue StandardError => e + failed_reverts << rename_info + say "Renaming #{type} from #{path_after_rename} back to "\ + "#{path_before_rename} failed. Review the error and try "\ + "again by running the `down` action. \n"\ + "#{e.message}: \n #{e.backtrace.join("\n")}" + end + end + + failed_reverts.each { |rename_info| redis.lpush(key, rename_info) } end + end - update_column_in_batches(:milestones, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end + def redis_key_for_type(type) + "rename:#{migration.name}:#{type}" end def file_storage? diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 2958ad4b8e5..05b86f32ce2 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -18,14 +18,20 @@ module Gitlab when :top_level MigrationClasses::Namespace.where(parent_id: nil) end - with_paths = MigrationClasses::Route.arel_table[:path]. - matches_any(path_patterns) + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) namespaces.joins(:route).where(with_paths) end def rename_namespace(namespace) old_full_path, new_full_path = rename_path_for_routable(namespace) + track_rename('namespace', old_full_path, new_full_path) + + rename_namespace_dependencies(namespace, old_full_path, new_full_path) + end + + def rename_namespace_dependencies(namespace, old_full_path, new_full_path) move_repositories(namespace, old_full_path, new_full_path) move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) @@ -33,6 +39,23 @@ module Gitlab remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) end + def revert_renames + reverts_for_type('namespace') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + namespace = MigrationClasses::Namespace.joins(:route) + .where(matches_path).first&.becomes(MigrationClasses::Namespace) + + if namespace + perform_rename(namespace, current_path, path_before_rename) + + rename_namespace_dependencies(namespace, current_path, path_before_rename) + else + say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\ + "namespace was renamed, or no longer exists at the expected path" + end + end + end + def rename_user(old_username, new_username) MigrationClasses::User.where(username: old_username) .update_all(username: new_username) @@ -52,15 +75,15 @@ module Gitlab end def repo_paths_for_namespace(namespace) - projects_for_namespace(namespace).distinct.select(:repository_storage). - map(&:repository_storage_path) + projects_for_namespace(namespace).distinct.select(:repository_storage) + .map(&:repository_storage_path) end def projects_for_namespace(namespace) namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) - namespace_or_children = MigrationClasses::Project. - arel_table[:namespace_id]. - in(namespace_ids) + namespace_or_children = MigrationClasses::Project + .arel_table[:namespace_id] + .in(namespace_ids) MigrationClasses::Project.where(namespace_or_children) end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 448717eb744..75a75f61953 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -16,12 +16,37 @@ module Gitlab def rename_project(project) old_full_path, new_full_path = rename_path_for_routable(project) + track_rename('project', old_full_path, new_full_path) + + move_project_folders(project, old_full_path, new_full_path) + end + + def move_project_folders(project, old_full_path, new_full_path) move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) end + def revert_renames + reverts_for_type('project') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + project = MigrationClasses::Project.joins(:route) + .where(matches_path).first + + if project + perform_rename(project, current_path, path_before_rename) + + move_project_folders(project, current_path, path_before_rename) + else + say "Couldn't rename project from #{current_path} back to "\ + "#{path_before_rename}, project was renamed or no longer "\ + "exists at the expected path." + + end + end + end + def move_repository(project, old_path, new_path) unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb new file mode 100644 index 00000000000..d9400e04b83 --- /dev/null +++ b/lib/gitlab/database/sha_attribute.rb @@ -0,0 +1,34 @@ +module Gitlab + module Database + BINARY_TYPE = if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + ActiveRecord::Type::Binary + end + + # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). + # + # Using ShaAttribute allows you to store SHA1 values as binary while still + # using them as if they were stored as string values. This gives you the + # ease of use of string values, but without the storage overhead. + class ShaAttribute < BINARY_TYPE + PACK_FORMAT = 'H*'.freeze + + # Casts binary data to a SHA1 in hexadecimal. + def type_cast_from_database(value) + value = super + + value ? value.unpack(PACK_FORMAT)[0] : nil + end + + # Casts a SHA1 in hexadecimal to the proper binary format. + def type_cast_for_database(value) + arg = value ? [value].pack(PACK_FORMAT) : nil + + super(arg) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 7bbd154eb03..d2360583741 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -52,7 +52,7 @@ module Gitlab # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| - marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| url = yield(text) diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index 2e197e5cd94..9c9620bc36a 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -6,7 +6,7 @@ module Gitlab private def link_dependencies - link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name| + link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=~!;\[]+)/) do |name| "https://pypi.python.org/pypi/#{name}" end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb deleted file mode 100644 index a78fde9d782..00000000000 --- a/lib/gitlab/devise_failure.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - class DeviseFailure < Devise::FailureApp - protected - - # Override `Devise::FailureApp#request_format` to handle a special case - # - # This tells Devise to handle an unauthenticated `.zip` request as an HTML - # request (i.e., redirect to sign in). - # - # Otherwise, Devise would respond with a 401 Unauthorized with - # `Content-Type: application/zip` and a response body in plaintext, and the - # browser would freak out. - # - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 - def request_format - if request.format == :zip - Mime::Type.lookup_by_extension(:html).ref - else - super - end - end - end -end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 7948782aecc..371cbe04b9b 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -37,6 +37,16 @@ module Gitlab def complete? start_sha && head_sha end + + def compare_in(project) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + project.commit(head_sha) + else + straight = start_sha == base_sha + CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + end + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 2aef7fdaa35..6d7de52cb80 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -5,7 +5,20 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, to: :diff, prefix: false + + # Finding a viewer for a diff file happens based only on extension and whether the + # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, + # and the order of these viewers doesn't really matter. + # + # However, when the diff file blobs are LFS pointers, we cannot know for sure whether the + # file being pointed to is binary or text. In this case, we match only on + # extension, preferring binary viewers over text ones if both exist, since the + # large files referred to in "Large File Storage" are much more likely to be + # binary than text. + RICH_VIEWERS = [ + DiffViewer::Image + ].sort_by { |v| v.binary? ? 0 : 1 }.freeze def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) @diff = diff @@ -58,19 +71,12 @@ module Gitlab diff_refs&.head_sha end - def content_sha - return old_content_sha if deleted_file? - return @content_sha if defined?(@content_sha) + def new_content_sha + return if deleted_file? + return @new_content_sha if defined?(@new_content_sha) refs = diff_refs || fallback_diff_refs - @content_sha = refs&.head_sha - end - - def content_commit - return @content_commit if defined?(@content_commit) - - sha = content_sha - @content_commit = repository.commit(sha) if sha + @new_content_sha = refs&.head_sha end def old_content_sha @@ -81,20 +87,13 @@ module Gitlab @old_content_sha = refs&.base_sha end - def old_content_commit - return @old_content_commit if defined?(@old_content_commit) - - sha = old_content_sha - @old_content_commit = repository.commit(sha) if sha - end - - def blob - return @blob if defined?(@blob) + def new_blob + return @new_blob if defined?(@new_blob) - sha = content_sha - return @blob = nil unless sha + sha = new_content_sha + return @new_blob = nil unless sha - repository.blob_at(sha, file_path) + @new_blob = repository.blob_at(sha, file_path) end def old_blob @@ -106,6 +105,14 @@ module Gitlab @old_blob = repository.blob_at(sha, old_path) end + def content_sha + new_content_sha || old_content_sha + end + + def blob + new_blob || old_blob + end + attr_writer :highlighted_diff_lines # Array of Gitlab::Diff::Line objects @@ -153,6 +160,112 @@ module Gitlab def file_identifier "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}" end + + def diffable? + repository.attributes(file_path).fetch('diff') { true } + end + + def binary? + old_blob&.binary? || new_blob&.binary? + end + + def text? + !binary? + end + + def external_storage_error? + old_blob&.external_storage_error? || new_blob&.external_storage_error? + end + + def stored_externally? + old_blob&.stored_externally? || new_blob&.stored_externally? + end + + def external_storage + old_blob&.external_storage || new_blob&.external_storage + end + + def content_changed? + old_blob && new_blob && old_blob.id != new_blob.id + end + + def different_type? + old_blob && new_blob && old_blob.binary? != new_blob.binary? + end + + def size + [old_blob&.size, new_blob&.size].compact.sum + end + + def raw_size + [old_blob&.raw_size, new_blob&.raw_size].compact.sum + end + + def raw_binary? + old_blob&.raw_binary? || new_blob&.raw_binary? + end + + def raw_text? + !raw_binary? && !different_type? + end + + def simple_viewer + @simple_viewer ||= simple_viewer_class.new(self) + end + + def rich_viewer + return @rich_viewer if defined?(@rich_viewer) + + @rich_viewer = rich_viewer_class&.new(self) + end + + def rendered_as_text?(ignore_errors: true) + simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) + end + + private + + def simple_viewer_class + return DiffViewer::NotDiffable unless diffable? + + if content_changed? + if raw_text? + DiffViewer::Text + else + DiffViewer::NoPreview + end + elsif new_file? + if raw_text? + DiffViewer::Text + else + DiffViewer::Added + end + elsif deleted_file? + if raw_text? + DiffViewer::Text + else + DiffViewer::Deleted + end + elsif renamed_file? + DiffViewer::Renamed + elsif mode_changed? + DiffViewer::ModeChanged + end + end + + def rich_viewer_class + viewer_class_from(RICH_VIEWERS) + end + + def viewer_class_from(classes) + return unless diffable? + return if different_type? || external_storage_error? + return unless new_file? || deleted_file? || content_changed? + + verify_binary = !stored_externally? + + classes.find { |viewer_class| viewer_class.can_render?(self, verify_binary: verify_binary) } + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 9a58b500a2c..fcda1fe2233 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -66,10 +66,7 @@ module Gitlab end def cacheable?(diff_file) - @merge_request_diff.present? && - diff_file.blob && - diff_file.blob.text? && - @project.repository.diffable?(diff_file.blob) + @merge_request_diff.present? && diff_file.text? && diff_file.diffable? end def cache_key diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index ed2f541977a..b669ee5b799 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -42,9 +42,9 @@ module Gitlab rich_line = if diff_line.unchanged? || diff_line.added? - new_lines[diff_line.new_pos - 1] + new_lines[diff_line.new_pos - 1]&.html_safe elsif diff_line.removed? - old_lines[diff_line.old_pos - 1] + old_lines[diff_line.old_pos - 1]&.html_safe end # Only update text if line is found. This will prevent @@ -60,13 +60,18 @@ module Gitlab end def old_lines - return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path) + @old_lines ||= highlighted_blob_lines(diff_file.old_blob) end def new_lines - return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path) + @new_lines ||= highlighted_blob_lines(diff_file.new_blob) + end + + def highlighted_blob_lines(blob) + return [] unless blob + + blob.load_all_data! + Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index bd52ae47e9f..2d89ccfc354 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -42,25 +42,25 @@ module Gitlab end def added? - type == 'new' || type == 'new-nonewline' + %w[new new-nonewline].include?(type) end def removed? - type == 'old' || type == 'old-nonewline' - end - - def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text - - @rich_text + %w[old old-nonewline].include?(type) end def meta? - type == 'match' + %w[match new-nonewline old-nonewline].include?(type) end def discussable? - !['match', 'new-nonewline', 'old-nonewline'].include?(type) + !meta? + end + + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text end def as_json(opts = nil) diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 481536a380b..0cb26fa45c8 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -14,16 +14,7 @@ module Gitlab lines = [] highlighted_diff_lines = diff_file.highlighted_diff_lines highlighted_diff_lines.each do |line| - if line.meta? || line.unchanged? - # line in the right panel is the same as in the left one - lines << { - left: line, - right: line - } - - free_right_index = nil - i += 1 - elsif line.removed? + if line.removed? lines << { left: line, right: nil @@ -51,6 +42,15 @@ module Gitlab free_right_index = nil i += 1 end + elsif line.meta? || line.unchanged? + # line in the right panel is the same as in the left one + lines << { + left: line, + right: line + } + + free_right_index = nil + i += 1 end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 4d96778a2b2..f80afb20f0c 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -145,23 +145,9 @@ module Gitlab private def find_diff_file(repository) - # We're at the initial commit, so just get that as we can't compare to anything. - compare = - if Gitlab::Git.blank_ref?(start_sha) - Gitlab::Git::Commit.find(repository.raw_repository, head_sha) - else - Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ) - end - - diff = compare.diffs(paths: paths).first - - return unless diff + return unless diff_refs.complete? - Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index dcabb5f7fe5..b68a1636814 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -216,7 +216,7 @@ module Gitlab def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths) + compare.diffs(paths: paths, expanded: true) end def position(diff_file, old_line, new_line) diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb index ab9537ed7d7..941244694e2 100644 --- a/lib/gitlab/downtime_check.rb +++ b/lib/gitlab/downtime_check.rb @@ -50,8 +50,8 @@ module Gitlab # Returns the class for the given migration file path. def class_for_migration_file(path) - File.basename(path, File.extname(path)).split('_', 2).last.camelize. - constantize + File.basename(path, File.extname(path)).split('_', 2).last.camelize + .constantize end # Returns true if the given migration can be performed without downtime. diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 38e27513281..72d7d4f84d1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -76,7 +76,7 @@ module Gitlab step( "Generating the patch against origin/master in #{patch_path}", - %w[git format-patch origin/master --stdout] + %w[git diff --binary origin/master...HEAD] ) do |output, status| throw(:halt_check, :ko) unless status.zero? @@ -134,7 +134,15 @@ module Gitlab step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", - %W[git apply --check --3way #{patch_path}] + # Don't use --check here because it can result in a 0-exit status even + # though the patch doesn't apply cleanly, e.g.: + # > git apply --check --3way foo.patch + # error: patch failed: lib/gitlab/ee_compat_check.rb:74 + # Falling back to three-way merge... + # Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts. + # > echo $? + # 0 + %W[git apply --3way #{patch_path}] ) do |output, status| puts output unless status.zero? @@ -149,6 +157,7 @@ module Gitlab status = 0 if failed_files.empty? end + command(%w[git reset --hard]) status end end @@ -172,8 +181,6 @@ module Gitlab end def find_merge_base_with_master(branch:) - return if merge_base_found? - # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # In total we go (20 + 54 + 148 + 403 = 625) commits deeper depth = 20 @@ -228,6 +235,10 @@ module Gitlab branch_name.parameterize << '.patch' end + def patch_url + "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/#{ENV['CI_JOB_ID']}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" + end + def step(desc, cmd = nil) puts "\n=> #{desc}\n" @@ -294,14 +305,11 @@ module Gitlab 2. Apply your branch's patch to EE - # In the CE repo - $ git fetch origin master - $ git format-patch origin/master --stdout > #{ce_branch}.patch - # In the EE repo $ git fetch origin master $ git checkout -b #{ee_branch_prefix} origin/master - $ git apply --3way path/to/#{ce_branch}.patch + $ wget #{patch_url} + $ git apply --3way #{ce_patch_name} At this point you might have conflicts such as: @@ -315,7 +323,7 @@ module Gitlab If the patch couldn't be applied cleanly, use the following command: # In the EE repo - $ git apply --reject path/to/#{ce_branch}.patch + $ git apply --reject #{ce_patch_name} This option makes git apply the parts of the patch that are applicable, and leave the rejected hunks in corresponding `.rej` files. @@ -328,7 +336,7 @@ module Gitlab # In the EE repo $ git push origin #{ee_branch_prefix} - ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. Once this is done, you can retry this failed build, and it should pass. diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index a4ca62bfc41..50559a48973 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -17,6 +17,13 @@ module Gitlab def filter_replies! document.xpath('//blockquote').each(&:remove) document.xpath('//table').each(&:remove) + + # bogus links with no href are sometimes added by outlook, + # and can result in Html2Text adding extra square brackets + # to the text, so we unwrap them here. + document.xpath('//a[not(@href)]').each do |link| + link.replace(link.children) + end end def filtered_html diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index ea035e33eff..cd9d3a6483f 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -4,7 +4,7 @@ module Gitlab class RepositoryPush attr_reader :author_id, :ref, :action - include Gitlab::Routing.url_helpers + include Gitlab::Routing include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project @@ -96,20 +96,13 @@ module Gitlab def target_url if @action == :push && commits if commits.length > 1 - namespace_project_compare_url(project_namespace, - project, - from: compare.start_commit, - to: compare.head_commit) + project_compare_url(project, from: compare.start_commit, to: compare.head_commit) else - namespace_project_commit_url(project_namespace, - project, - commits.first) + project_commit_url(project, commits.first) end else unless @action == :delete - namespace_project_tree_url(project_namespace, - project, - ref_name) + project_tree_url(project, ref_name) end end end @@ -124,7 +117,7 @@ module Gitlab def subject subject_text = '[Git]' - subject_text << "[#{project.path_with_namespace}]" + subject_text << "[#{project.full_path}]" subject_text << "[#{ref_name}]" if @action == :push subject_text << ' ' diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 270d67dd50c..1d6f5bb5e1c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -6,12 +6,13 @@ module Gitlab end def call(env) - route = Gitlab::EtagCaching::Router.match(env) + request = Rack::Request.new(env) + route = Gitlab::EtagCaching::Router.match(request.path_info) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(env) + etag, cached_value_present = get_etag(request) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -27,8 +28,8 @@ module Gitlab private - def get_etag(env) - cache_key = env['PATH_INFO'] + def get_etag(request) + cache_key = request.path store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index ca49eda51fb..75167a6b088 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -53,8 +53,8 @@ module Gitlab ) ].freeze - def self.match(env) - ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + def self.match(path) + ROUTES.find { |route| route.regexp.match(path) } end end end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 0039fc01c8f..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,8 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) - "#{REDIS_NAMESPACE}#{key}" + def redis_shared_state_key(key) + raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) + + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 62ddd45785d..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -10,25 +10,33 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease - LUA_CANCEL_SCRIPT = <<-EOS.freeze + LUA_CANCEL_SCRIPT = <<~EOS.freeze local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then redis.call("del", key) end EOS + LUA_RENEW_SCRIPT = <<~EOS.freeze + local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2] + if redis.call("get", key) == uuid then + redis.call("expire", key, ttl) + return uuid + end + EOS + def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -37,15 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + end + end + + # Try to renew an existing lease. Return lease UUID on success, + # false if the lease is taken by a different UUID or inexistent. + def renew + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) + result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb new file mode 100644 index 00000000000..bb14a8cd9e7 --- /dev/null +++ b/lib/gitlab/fake_application_settings.rb @@ -0,0 +1,27 @@ +# This class extends an OpenStruct object by adding predicate methods to mimic +# ActiveRecord access. We rely on the initial values being true or false to +# determine whether to define a predicate method because for a newly-added +# column that has not been migrated yet, there is no way to determine the +# column type without parsing db/schema.rb. +module Gitlab + class FakeApplicationSettings < OpenStruct + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + # Mimic ActiveRecord predicate methods for boolean values + def self.define_predicate_methods(options) + options.each do |key, value| + next if key.to_s.end_with?('?') + next unless [true, false].include?(value) + + define_method "#{key}?" do + actual_key = key.to_s.chomp('?') + self[actual_key] + end + end + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 936606152e9..b6449f27034 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -7,8 +7,10 @@ module Gitlab CommandError = Class.new(StandardError) class << self + include Gitlab::EncodingHelper + def ref_name(ref) - ref.sub(/\Arefs\/(tags|heads)\//, '') + encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..0deaab01b5b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC for #load_blame. + module Gitlab module Git class Blame @@ -24,6 +26,7 @@ module Gitlab private + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) # Read in binary mode to ensure ASCII-8BIT diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index d60e607b02b..db6cfc9671f 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -14,6 +16,47 @@ module Gitlab class << self def find(repository, sha, path) + Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| + if is_enabled + find_by_gitaly(repository, sha, path) + else + find_by_rugged(repository, sha, path) + end + end + end + + def find_by_gitaly(repository, sha, path) + path = path.sub(/\A\/*/, '') + path = '/' if path.empty? + name = File.basename(path) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + return unless entry + + case entry.type + when :COMMIT + new( + id: entry.oid, + name: name, + size: 0, + data: '', + path: path, + commit_id: sha + ) + when :BLOB + new( + id: entry.oid, + name: name, + size: entry.size, + data: entry.data.dup, + mode: entry.mode.to_s(8), + path: path, + commit_id: sha, + binary: binary?(entry.data) + ) + end + end + + def find_by_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree @@ -42,16 +85,32 @@ module Gitlab end def raw(repository, sha) - blob = repository.lookup(sha) + Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + else + blob = repository.lookup(sha) - new( - id: blob.oid, - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - binary: blob.binary? - ) + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + end end + def binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary + end + + private + # Recursive search of blob id by path # # Ex. @@ -120,15 +179,29 @@ module Gitlab return if @data == '' # don't mess with submodule blobs return @data if @loaded_all_data + Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + @data = begin + if is_enabled + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + else + repository.lookup(id).content + end + end + end + @loaded_all_data = true - @data = repository.lookup(id).content @loaded_size = @data.bytesize + @binary = nil end def name encode! @name end + def path + encode! @path + end + def truncated? size && (size > loaded_size) end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index bb04731f08c..ca7e3a7c4be 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -4,7 +4,7 @@ module Gitlab class Commit include Gitlab::EncodingHelper - attr_accessor :raw_commit, :head, :refs + attr_accessor :raw_commit, :head SERIALIZE_KEYS = [ :id, :message, :parent_ids, @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -97,16 +98,89 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.commits_between(base, head).map do |commit| - decorate(commit) + Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.between(base, head) + else + repo.commits_between(base, head).map { |c| decorate(c) } + end end rescue Rugged::ReferenceError [] end - # Delegate Repository#find_commits + # Returns commits collection + # + # Ex. + # Commit.find_all( + # repo, + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - repo.find_commits(options) + Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| + if is_enabled + find_all_by_gitaly(repo, options) + else + find_all_by_rugged(repo, options) + end + end + end + + def find_all_by_rugged(repo, options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + rugged = repo.rugged + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + walker.sorting(rugged_sort_type(actual_options[:order])) + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + commits.push(decorate(commit)) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + def find_all_by_gitaly(repo, options = {}) + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(commit, ref = nil) @@ -131,15 +205,32 @@ module Gitlab diff.find_similar!(break_rewrites: break_rewrites) diff end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE | Rugged::SORT_TOPO + } + + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) + end end def initialize(raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit - if raw_commit.is_a?(Hash) + case raw_commit + when Hash init_from_hash(raw_commit) - elsif raw_commit.is_a?(Rugged::Commit) + when Rugged::Commit init_from_rugged(raw_commit) + when Gitlab::GitalyClient::Commit + init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end @@ -175,8 +266,10 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. - def to_diff(options = {}) - diff_from_parent(options).patch + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 + def to_diff + diff_from_parent.patch end # Returns a diff object for the changes from this commit's first parent. @@ -216,7 +309,23 @@ module Gitlab end def parents - raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + case raw_commit + when Rugged::Commit + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + when Gitlab::GitalyClient::Commit + parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact + else + raise NotImplementedError, "commit source doesn't support #parents" + end + end + + # Get the gpg signature of this commit. + # + # Ex. + # commit.signature(repo) + # + def signature(repo) + Rugged::Commit.extract_signature(repo.rugged, sha) end def stats @@ -227,7 +336,7 @@ module Gitlab begin raw_commit.to_mbox(options) rescue Rugged::InvalidError => ex - if ex.message =~ /Commit \w+ is a merge commit/ + if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' end end @@ -299,6 +408,22 @@ module Gitlab @parent_ids = commit.parents.map(&:oid) end + def init_from_gitaly(commit) + @raw_commit = commit + @id = commit.id + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + @message = (commit.body.presence || commit.subject).dup + @authored_date = Time.at(commit.author.date.seconds) + @author_name = commit.author.name.dup + @author_email = commit.author.email.dup + @committed_date = Time.at(commit.committer.date.seconds) + @committer_name = commit.committer.name.dup + @committer_email = commit.committer.email.dup + @parent_ids = commit.parent_ids + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 696a2acd5e3..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,9 +1,11 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare attr_reader :head, :base, :straight - def initialize(repository, base, head, straight = false) + def initialize(repository, base, head, straight: false) @repository = repository @straight = straight diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 8926aa19925..9e00abefd02 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -16,13 +18,32 @@ module Gitlab alias_method :renamed_file?, :renamed_file attr_accessor :expanded + attr_writer :too_large + + alias_method :expanded?, :expanded - # We need this accessor because of `to_hash` and `init_from_hash` - attr_accessor :too_large + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self # The maximum size of a diff to display. def size_limit + if RequestStore.active? + RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit + else + find_size_limit + end + end + + # The maximum size before a diff is collapsed. + def collapse_limit + if RequestStore.active? + RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit + else + find_collapse_limit + end + end + + def find_size_limit if Feature.enabled?('gitlab_git_diff_size_limit_increase') 200.kilobytes else @@ -30,8 +51,7 @@ module Gitlab end end - # The maximum size before a diff is collapsed. - def collapse_limit + def find_collapse_limit if Feature.enabled?('gitlab_git_diff_size_limit_increase') 100.kilobytes else @@ -63,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options @@ -213,16 +139,10 @@ module Gitlab end end - def serialize_keys - @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) - end - def to_hash hash = {} - keys = serialize_keys - - keys.each do |key| + SERIALIZE_KEYS.each do |key| hash[key] = send(key) end @@ -249,6 +169,9 @@ module Gitlab end end + # This is used by `to_hash` and `init_from_hash`. + alias_method :too_large, :too_large? + def too_large! @diff = '' @line_count = 0 @@ -297,13 +220,13 @@ module Gitlab def init_from_hash(hash) raw_diff = hash.symbolize_keys - serialize_keys.each do |key| + SERIALIZE_KEYS.each do |key| send(:"#{key}=", raw_diff[key.to_sym]) end end def init_from_gitaly(diff) - @diff = diff.patch if diff.respond_to?(:patch) + @diff = encode!(diff.patch) if diff.respond_to?(:patch) @new_path = encode!(diff.to_path.dup) @old_path = encode!(diff.from_path.dup) @a_mode = diff.old_mode.to_s(8) @@ -311,6 +234,8 @@ module Gitlab @new_file = diff.from_id == BLANK_SHA @renamed_file = diff.from_path != diff.to_path @deleted_file = diff.to_id == BLANK_SHA + + collapse! if diff.respond_to?(:collapsed) && diff.collapsed end def prune_diff_if_eligible diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 334e06a6eca..87ed9c3ea26 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection @@ -5,16 +7,28 @@ module Gitlab DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + attr_reader :limits + + delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + + def self.collection_limits(options = {}) + limits = {} + limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file + limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + + OpenStruct.new(limits) + end + def initialize(iterator, options = {}) @iterator = iterator - @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file - @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min - @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @limits = self.class.collection_limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) + @from_gitaly = options.fetch(:from_gitaly, false) @line_count = 0 @byte_count = 0 @@ -24,9 +38,23 @@ module Gitlab end def each(&block) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) + @array.each(&block) + + return if @overflow + return if @iterator.nil? + + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + if is_enabled && @from_gitaly + each_gitaly_patch(&block) + else + each_rugged_patch(&block) + end end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end def empty? @@ -72,23 +100,32 @@ module Gitlab end def over_safe_limits?(files) - files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes end - def each_patch - i = 0 - @array.each do |diff| - yield diff + def each_gitaly_patch + i = @array.length + + @iterator.each do |raw| + diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded) + + if raw.overflow_marker + @overflow = true + break + end + + yield @array[i] = diff i += 1 end + end - return if @overflow - return if @iterator.nil? + def each_rugged_patch + i = @array.length @iterator.each do |raw| @empty = false - if @enforce_limits && i >= @max_files + if @enforce_limits && i >= max_files @overflow = true break end @@ -97,14 +134,14 @@ module Gitlab diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if !expanded && over_safe_limits?(i) + if !expanded && over_safe_limits?(i) && diff.line_count > 0 diff.collapse! end @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -114,11 +151,6 @@ module Gitlab yield @array[i] = diff i += 1 end - - @populated = true - - # Allow iterator to be garbage-collected. It cannot be reused anyway. - @iterator = nil end end end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb new file mode 100644 index 00000000000..4a43b9b444d --- /dev/null +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -0,0 +1,79 @@ +# Gitaly note: JV: no RPC's here. + +module Gitlab + module Git + class GitmodulesParser + def initialize(content) + @content = content + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information, indexed by path. + def parse + reindex_by_path(get_submodules_by_name) + end + + private + + class State + def initialize + @result = {} + @current_submodule = nil + end + + def start_section(section) + # In some .gitmodules files (e.g. nodegit's), a header + # with the same name appears multiple times; we want to + # accumulate the configs across these + @current_submodule = @result[section] || { 'name' => section } + @result[section] = @current_submodule + end + + def set_attribute(attr, value) + @current_submodule[attr] = value + end + + def section_started? + !@current_submodule.nil? + end + + def submodules_by_name + @result + end + end + + def get_submodules_by_name + iterator = State.new + + @content.split("\n").each_with_object(iterator) do |text, iterator| + next if text =~ /^\s*#/ + + if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ + iterator.start_section($~[:name]) + else + next unless iterator.section_started? + + next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ + + value = $~[:value].chomp + iterator.set_attribute($~[:key], value) + end + end + + iterator.submodules_by_name + end + + def reindex_by_path(submodules_by_name) + # Convert from an indexed by name to an array indexed by path + # If a submodule doesn't have a path, it is considered bogus + # and is ignored + submodules_by_name.each_with_object({}) do |(name, data), results| + path = data.delete 'path' + next unless path + + results[path] = data + end + end + end + end +end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index bd90d24a2ec..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,12 +1,17 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook GL_PROTOCOL = 'web'.freeze attr_reader :name, :repo_path, :path - def initialize(name, repo_path) + def initialize(name, project) @name = name - @repo_path = repo_path + @project = project + @repo_path = project.repository.path @path = File.join(repo_path.strip, 'hooks', name) end @@ -38,7 +43,8 @@ module Gitlab vars = { 'GL_ID' => gl_id, 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL + 'GL_PROTOCOL' => GL_PROTOCOL, + 'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false) } options = { diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 1add037fa5f..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index @@ -110,10 +114,6 @@ module Gitlab if segment == '..' raise IndexError, 'Path cannot include directory traversal' end - - unless segment =~ Gitlab::Regex.file_name_regex - raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" - end end pathname.to_s diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9d6adbdb4ac..70d793f8e4a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,6 +45,8 @@ module Gitlab :bare?, to: :rugged + delegate :exists?, to: :gitaly_repository_client + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -80,16 +82,14 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch + def branches + gitaly_migrate(:branches) do |is_enabled| + if is_enabled + gitaly_ref_client.branches + else + branches_filter end - end.compact - - sort_branches(branches, sort_by) + end end def reload_rugged @@ -107,17 +107,18 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) gitaly_migrate(:local_branches) do |is_enabled| if is_enabled - gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch| - Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch) - end + gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -164,20 +165,16 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end + gitaly_migrate(:tags) do |is_enabled| + if is_enabled + tags_from_gitaly + else + tags_from_rugged end - - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) - end.sort_by(&:name) + end end # Returns true if the given tag exists @@ -206,21 +203,10 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end - def repo_exists? - !!rugged - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -299,28 +285,6 @@ module Gitlab (size.to_f / 1024).round(2) end - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) - end - - greps - end - # Use the Rugged Walker API to build an array of commits. # # Usage. @@ -333,85 +297,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -456,7 +345,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -464,6 +353,13 @@ module Gitlab rugged.merge_base(from, to) end + # Gitaly note: JV: check gitlab-ee before removing this method. + def rugged_is_ancestor?(ancestor_id, descendant_id) + return false if ancestor_id.nil? || descendant_id.nil? + + merge_base_commit(ancestor_id, descendant_id) == ancestor_id + end + # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) gitaly_commit_client.is_ancestor(from, to) @@ -494,70 +390,6 @@ module Gitlab end end - # Returns commits collection - # - # Ex. - # repo.find_commits( - # ref: 'master', - # max_count: 10, - # skip: 5, - # order: :date - # ) - # - # +options+ is a Hash of optional arguments to git - # :ref is the ref from which to begin (SHA1 or name) - # :contains is the commit contained by the refs from which to begin (SHA1 or name) - # :max_count is the maximum number of commits to fetch - # :skip is the number of commits to skip - # :order is the commits order and allowed value is :none (default), :date, - # :topo, or any combination of them (in an array). Commit ordering types - # are documented here: - # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) - # - def find_commits(options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :contains, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) - end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - elsif actual_options[:contains] - branches_contains(actual_options[:contains]).each do |branch| - walker.push(branch.target_id) - end - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end - end - - sort_type = rugged_sort_type(actual_options[:order]) - walker.sorting(sort_type) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - gitlab_commit = Gitlab::Git::Commit.decorate(commit) - commits.push(gitlab_commit) - end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - # Returns branch names collection that contains the special commit(SHA1 # or name) # @@ -613,56 +445,43 @@ module Gitlab rugged.rev_parse(oid_or_ref_name) end - # Return hash with submodules info for this repository + # Returns url for submodule # # Ex. - # { - # "rack" => { - # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", - # "path" => "rack", - # "url" => "git://github.com/chneukirchen/rack.git" - # }, - # "encoding" => { - # "id" => .... - # } - # } + # @repository.submodule_url_for('master', 'rack') + # # => git@localhost:rack.git # - def submodules(ref) - commit = rev_parse_target(ref) - return {} unless commit - - begin - content = blob_content(commit, ".gitmodules") - rescue InvalidBlobName - return {} + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 + def submodule_url_for(ref, path) + Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| + if is_enabled + gitaly_submodule_url_for(ref, path) + else + if submodules(ref).any? + submodule = submodules(ref)[path] + submodule['url'] if submodule + end + end end - - parse_gitmodules(commit, content) end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + gitaly_migrate(:commit_count) do |is_enabled| + if is_enabled + gitaly_commit_client.commit_count(ref) + else + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + end end - # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or - # tag name or a commit SHA. Valid +reset_type+ values are: - # - # [:soft] - # the head will be moved to the commit. - # [:mixed] - # will trigger a +:soft+ reset, plus the index will be replaced - # with the content of the commit tree. - # [:hard] - # will trigger a +:mixed+ reset and the working directory will be - # replaced with the content of the index. (Untracked and ignored files - # will be left alone) - delegate :reset, to: :rugged - # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # @@ -687,154 +506,6 @@ module Gitlab # TODO: implement this method end - # Check out the specified ref. Valid options are: - # - # :b - Create a new branch at +start_point+ and set HEAD to the new - # branch. - # - # * These options are passed to the Rugged::Repository#checkout method: - # - # :progress :: - # A callback that will be executed for checkout progress notifications. - # Up to 3 parameters are passed on each execution: - # - # - The path to the last updated file (or +nil+ on the very first - # invocation). - # - The number of completed checkout steps. - # - The number of total checkout steps to be performed. - # - # :notify :: - # A callback that will be executed for each checkout notification - # types specified with +:notify_flags+. Up to 5 parameters are passed - # on each execution: - # - # - An array containing the +:notify_flags+ that caused the callback - # execution. - # - The path of the current file. - # - A hash describing the baseline blob (or +nil+ if it does not - # exist). - # - A hash describing the target blob (or +nil+ if it does not exist). - # - A hash describing the workdir blob (or +nil+ if it does not - # exist). - # - # :strategy :: - # A single symbol or an array of symbols representing the strategies - # to use when performing the checkout. Possible values are: - # - # :none :: - # Perform a dry run (default). - # - # :safe :: - # Allow safe updates that cannot overwrite uncommitted data. - # - # :safe_create :: - # Allow safe updates plus creation of missing files. - # - # :force :: - # Allow all updates to force working directory to look like index. - # - # :allow_conflicts :: - # Allow checkout to make safe updates even if conflicts are found. - # - # :remove_untracked :: - # Remove untracked files not in index (that are not ignored). - # - # :remove_ignored :: - # Remove ignored files not in index. - # - # :update_only :: - # Only update existing files, don't create new ones. - # - # :dont_update_index :: - # Normally checkout updates index entries as it goes; this stops - # that. - # - # :no_refresh :: - # Don't refresh index/config/etc before doing checkout. - # - # :disable_pathspec_match :: - # Treat pathspec as simple list of exact match file paths. - # - # :skip_locked_directories :: - # Ignore directories in use, they will be left empty. - # - # :skip_unmerged :: - # Allow checkout to skip unmerged files (NOT IMPLEMENTED). - # - # :use_ours :: - # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). - # - # :use_theirs :: - # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). - # - # :update_submodules :: - # Recursively checkout submodules with same options (NOT - # IMPLEMENTED). - # - # :update_submodules_if_changed :: - # Recursively checkout submodules if HEAD moved in super repo (NOT - # IMPLEMENTED). - # - # :disable_filters :: - # If +true+, filters like CRLF line conversion will be disabled. - # - # :dir_mode :: - # Mode for newly created directories. Default: +0755+. - # - # :file_mode :: - # Mode for newly created files. Default: +0755+ or +0644+. - # - # :file_open_flags :: - # Mode for opening files. Default: - # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. - # - # :notify_flags :: - # A single symbol or an array of symbols representing the cases in - # which the +:notify+ callback should be invoked. Possible values are: - # - # :none :: - # Do not invoke the +:notify+ callback (default). - # - # :conflict :: - # Invoke the callback for conflicting paths. - # - # :dirty :: - # Invoke the callback for "dirty" files, i.e. those that do not need - # an update but no longer match the baseline. - # - # :updated :: - # Invoke the callback for any file that was changed. - # - # :untracked :: - # Invoke the callback for untracked files. - # - # :ignored :: - # Invoke the callback for ignored files. - # - # :all :: - # Invoke the callback for all these cases. - # - # :paths :: - # A glob string or an array of glob strings specifying which paths - # should be taken into account for the checkout operation. +nil+ will - # match all files. Default: +nil+. - # - # :baseline :: - # A Rugged::Tree that represents the current, expected contents of the - # workdir. Default: +HEAD+. - # - # :target_directory :: - # A path to an alternative workdir directory in which the checkout - # should be performed. - def checkout(ref, options = {}, start_point = "HEAD") - if options[:b] - rugged.branches.create(ref, start_point) - options.delete(:b) - end - default_options = { strategy: [:recreate_missing, :safe] } - rugged.checkout(ref, default_options.merge(options)) - end - # Delete the specified branch from the repository def delete_branch(branch_name) rugged.branches.delete(branch_name) @@ -847,7 +518,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -906,6 +578,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -932,6 +605,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -962,11 +636,6 @@ module Gitlab end end - # Checks if the blob should be diffable according to its attributes - def diffable?(blob) - attributes(blob.path).fetch('diff') { blob.text? } - end - # Returns the Git attributes for the given file path. # # See `Gitlab::Git::Attributes` for more information. @@ -974,12 +643,177 @@ module Gitlab @attributes.attributes(path) end + def languages(ref = nil) + Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| + if is_enabled + gitaly_commit_client.languages(ref) + else + ref ||= rugged.head.target_id + languages = Linguist::Repository.new(rugged, ref).languages + total = languages.map(&:last).sum + + languages = languages.map do |language| + name, share = language + color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" + { + value: (share.to_f * 100 / total).round(2), + label: name, + color: color, + highlight: color + } + end + + languages.sort do |x, y| + y[:value] <=> x[:value] + end + end + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) + end + + def gitaly_repository_client + @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) + end + private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + + # We are trying to deprecate this method because it does a lot of work + # but it seems to be used only to look up submodule URL's. + # https://gitlab.com/gitlab-org/gitaly/issues/329 + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parser = GitmodulesParser.new(content) + fill_submodule_ids(commit, parser.parse) + end + + def gitaly_submodule_url_for(ref, path) + # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. + commit_object = gitaly_commit_client.tree_entry(ref, path, 1) + + return unless commit_object && commit_object.type == :COMMIT + + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + found_module = GitmodulesParser.new(gitmodules.data).parse[path] + + found_module && found_module['url'] + end + def alternate_object_directories Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact end @@ -1003,42 +837,19 @@ module Gitlab end end - # Parses the contents of a .gitmodules file and returns a hash of - # submodule information. - def parse_gitmodules(commit, content) - modules = {} - - name = nil - content.each_line do |line| - case line.strip - when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header - name = $~[:name] - modules[name] = {} - when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair - key = $~[:key] - value = $~[:value].chomp - - next unless name && modules[name] - - modules[name][key] = value - - if key == 'path' - begin - modules[name]['id'] = blob_content(commit, value) - rescue InvalidBlobName - # The current entry is invalid - modules.delete(name) - name = nil - end - end - when /\A#/ # Comment - next - else # Invalid line - name = nil + # Fill in the 'id' field of a submodule hash from its values + # as-of +commit+. Return a Hash consisting only of entries + # from the submodule hash for which the 'id' field is filled. + def fill_submodule_ids(commit, submodule_data) + submodule_data.each do |path, data| + id = begin + blob_content(commit, path) + rescue InvalidBlobName + nil end + data['id'] = id end - - modules + submodule_data.select { |path, data| data['id'] } end # Returns true if +commit+ introduced changes to +path+, using commit @@ -1145,73 +956,6 @@ module Gitlab index end - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1240,12 +984,25 @@ module Gitlab end end - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) + def tags_from_rugged + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + end.sort_by(&:name) end - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + def tags_from_gitaly + gitaly_ref_client.tags end def gitaly_migrate(method, &block) @@ -1255,20 +1012,6 @@ module Gitlab rescue GRPC::BadStatus => e raise CommandError.new(e) end - - # Returns the `Rugged` sorting type constant for one or more given - # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array - # containing more than one of them. `:date` uses a combination of date and - # topological sorting to closer mimic git's native ordering. - def rugged_sort_type(sort_type) - @rugged_sort_types ||= { - none: Rugged::SORT_NONE, - topo: Rugged::SORT_TOPO, - date: Rugged::SORT_DATE | Rugged::SORT_TOPO - } - - @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) - end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b9afa05c819..8e959c57c7c 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,36 +12,23 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) + Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| + if is_enabled + client = Gitlab::GitalyClient::CommitService.new(repository) + client.tree_entries(repository, sha, path) + else + tree_entries_from_rugged(repository, sha, path) + end end end + private + # Recursive search of tree id for path # # Ex. @@ -68,6 +57,34 @@ module Gitlab entry[:oid] end end + + def tree_entries_from_rugged(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode].to_s(8), + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha + ) + end + end end def initialize(options) @@ -80,6 +97,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def dir? type == :tree end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 0a19d24eb20..0b62911958d 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -22,12 +22,13 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities + attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil) @actor = actor @project = project @protocol = protocol + @redirected_path = redirected_path @authentication_abilities = authentication_abilities end @@ -35,6 +36,7 @@ module Gitlab check_protocol! check_active_user! check_project_accessibility! + check_project_moved! check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! @@ -87,6 +89,21 @@ module Gitlab end end + def check_project_moved! + if redirected_path + url = protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + message = <<-MESSAGE.strip_heredoc + Project '#{redirected_path}' was moved to '#{project.full_path}'. + + Please update your Git remote and try again: + + git remote set-url origin #{url} + MESSAGE + + raise NotFoundError, message + end + end + def check_command_disabled!(cmd) if upload_pack?(cmd) check_upload_pack_disabled! diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 2343446bf22..c90ef282fdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -1,3 +1,5 @@ +require 'base64' + require 'gitaly' module Gitlab @@ -48,12 +50,28 @@ module Gitlab address end - def self.enabled? - Gitlab.config.gitaly.enabled + # All Gitaly RPC call sites should use GitalyClient.call. This method + # makes sure that per-request authentication headers are set. + def self.call(storage, service, rpc, request) + metadata = request_metadata(storage) + metadata = yield(metadata) if block_given? + stub(service, storage).send(rpc, request, metadata) + end + + def self.request_metadata(storage) + encoded_token = Base64.strict_encode64(token(storage).to_s) + { metadata: { 'authorization' => "Bearer #{encoded_token}" } } + end + + def self.token(storage) + params = Gitlab.config.repositories.storages[storage] + raise "storage not found: #{storage.inspect}" if params.nil? + + params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) - return false if !enabled? || status == MigrationStatus::DISABLED + return false if status == MigrationStatus::DISABLED feature = Feature.get("gitaly_#{feature}") @@ -68,8 +86,8 @@ module Gitlab feature.enabled? end - def self.migrate(feature) - is_enabled = feature_enabled?(feature) + def self.migrate(feature, status: MigrationStatus::OPT_IN) + is_enabled = feature_enabled?(feature, status: status) metric_name = feature.to_s metric_name += "_gitaly" if is_enabled diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb new file mode 100644 index 00000000000..7ea8e8d0857 --- /dev/null +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -0,0 +1,30 @@ +module Gitlab + module GitalyClient + class BlobService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + end + + def get_blob(oid:, limit:) + request = Gitaly::GetBlobRequest.new( + repository: @gitaly_repo, + oid: oid, + limit: limit + ) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + + blob = response.first + return unless blob.oid.present? + + data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + + Gitlab::Git::Blob.new( + id: blob.oid, + size: blob.size, + data: data, + binary: Gitlab::Git::Blob.binary?(data) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index ba3da781dad..61fe462d762 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -1,58 +1,13 @@ module Gitlab module GitalyClient class Commit - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + attr_reader :repository, :gitaly_commit - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @repository = repository - end - - def is_ancestor(ancestor_id, child_id) - stub = GitalyClient.stub(:commit, @repository.storage) - request = Gitaly::CommitIsAncestorRequest.new( - repository: @gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) - - stub.commit_is_ancestor(request).value - end - - def diff_from_parent(commit, options = {}) - request_params = commit_diff_request_params(commit, options) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - - response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) - Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) - end + delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit - def commit_deltas(commit) - request_params = commit_diff_request_params(commit) - - response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) - response.flat_map do |msg| - msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } - end - end - - private - - def commit_diff_request_params(commit, options = {}) - parent_id = commit.parents[0]&.id || EMPTY_TREE_ID - - { - repository: @gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - paths: options.fetch(:paths, []) - } - end - - def diff_service_stub - GitalyClient.stub(:diff, @repository.storage) + def initialize(repository, gitaly_commit) + @repository = repository + @gitaly_commit = gitaly_commit end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb new file mode 100644 index 00000000000..a834781b1f1 --- /dev/null +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -0,0 +1,151 @@ +module Gitlab + module GitalyClient + class CommitService + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + request_params[:enforce_limits] = options.fetch(:limits, true) + request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + + request = Gitaly::CommitDiffRequest.new(request_params) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true)) + end + + def commit_deltas(commit) + request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } + end + end + + def tree_entry(ref, path, limit = nil) + request = Gitaly::TreeEntryRequest.new( + repository: @gitaly_repo, + revision: ref, + path: path.dup.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + entry = response.first + return unless entry.oid.present? + + if entry.type == :BLOB + rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } + entry.data += rest_of_data + end + + entry + end + + def tree_entries(repository, revision, path) + request = Gitaly::GetTreeEntriesRequest.new( + repository: @gitaly_repo, + revision: revision, + path: path.presence || '.' + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + + response.flat_map do |message| + message.entries.map do |gitaly_tree_entry| + entry_path = gitaly_tree_entry.path.dup + Gitlab::Git::Tree.new( + id: gitaly_tree_entry.oid, + root_id: gitaly_tree_entry.root_oid, + type: gitaly_tree_entry.type.downcase, + mode: gitaly_tree_entry.mode.to_s(8), + name: File.basename(entry_path), + path: entry_path, + commit_id: gitaly_tree_entry.commit_oid + ) + end + end + end + + def commit_count(ref) + request = Gitaly::CountCommitsRequest.new( + repository: @gitaly_repo, + revision: ref + ) + + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + end + + def between(from, to) + request = Gitaly::CommitsBetweenRequest.new( + repository: @gitaly_repo, + from: from, + to: to + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + consume_commits_response(response) + end + + def find_all_commits(opts = {}) + request = Gitaly::FindAllCommitsRequest.new( + repository: @gitaly_repo, + revision: opts[:ref].to_s, + max_count: opts[:max_count].to_i, + skip: opts[:skip].to_i + ) + request.order = opts[:order].upcase if opts[:order].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + consume_commits_response(response) + end + + def languages(ref = nil) + request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') + response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) + + response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } + end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def consume_commits_response(response) + response.flat_map do |message| + message.commits.map do |gitaly_commit| + commit = GitalyClient::Commit.new(@repository, gitaly_commit) + Gitlab::Git::Commit.new(commit) + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 1e117b7e74a..d459c9a88fb 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,7 +1,7 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze attr_accessor(*FIELDS) diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index d84e8d752dc..65d81dc5d46 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -13,7 +13,10 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) - diff_params[:patch] = diff_msg.raw_patch_data + # gRPC uses frozen strings by default, and we need to have an unfrozen string as it + # gets processed further down the line. So we unfreeze the first chunk of the patch + # in case it's the only chunk we receive for this diff. + diff_params[:patch] = diff_msg.raw_patch_data.dup current_diff = GitalyClient::Diff.new(diff_params) else diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb new file mode 100644 index 00000000000..326e6f7dafc --- /dev/null +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -0,0 +1,20 @@ +module Gitlab + module GitalyClient + class NotificationService + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def post_receive + GitalyClient.call( + @storage, + :notification_service, + :post_receive, + Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb deleted file mode 100644 index 719554eac52..00000000000 --- a/lib/gitlab/gitaly_client/notifications.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module GitalyClient - class Notifications - attr_accessor :stub - - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @stub = GitalyClient.stub(:notifications, repository.storage) - end - - def post_receive - request = Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) - @stub.post_receive(request) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb deleted file mode 100644 index 227fe45642e..00000000000 --- a/lib/gitlab/gitaly_client/ref.rb +++ /dev/null @@ -1,72 +0,0 @@ -module Gitlab - module GitalyClient - class Ref - attr_accessor :stub - - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @stub = GitalyClient.stub(:ref, repository.storage) - end - - def default_branch_name - request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - branch_name = stub.find_default_branch_name(request).name - - Gitlab::Git.branch_name(branch_name) - end - - def branch_names - request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - consume_refs_response(stub.find_all_branch_names(request), prefix: 'refs/heads/') - end - - def tag_names - request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') - end - - def find_ref_name(commit_id, ref_prefix) - request = Gitaly::FindRefNameRequest.new( - repository: @gitaly_repo, - commit_id: commit_id, - prefix: ref_prefix - ) - - stub.find_ref_name(request).name - end - - def count_tag_names - tag_names.count - end - - def count_branch_names - branch_names.count - end - - def local_branches(sort_by: nil) - request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) - request.sort_by = sort_by_param(sort_by) if sort_by - consume_branches_response(stub.find_local_branches(request)) - end - - private - - def consume_refs_response(response, prefix:) - response.flat_map do |r| - r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } - end - end - - def sort_by_param(sort_by) - enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) - raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value - enum_value - end - - def consume_branches_response(response) - response.flat_map { |r| r.branches } - end - end - end -end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb new file mode 100644 index 00000000000..b0f7548b7dc --- /dev/null +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -0,0 +1,148 @@ +module Gitlab + module GitalyClient + class RefService + include Gitlab::EncodingHelper + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def branches + request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + response.flat_map do |message| + message.branches.map do |branch| + gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) + target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) + Gitlab::Git.branch_name(response.name) + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) + consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) + consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } + end + + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @gitaly_repo, + commit_id: commit_id, + prefix: ref_prefix + ) + encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + end + + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) + consume_branches_response(response) + end + + def tags + request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + consume_tags_response(response) + end + + private + + def consume_refs_response(response) + response.flat_map { |message| message.names.map { |name| yield(name) } } + end + + def sort_by_param(sort_by) + sort_by = 'name' if sort_by == 'name_asc' + + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map do |message| + message.branches.map do |gitaly_branch| + Gitlab::Git::Branch.new( + @repository, + encode!(gitaly_branch.name.dup), + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) + ) + end + end + end + + def consume_tags_response(response) + response.flat_map do |message| + message.tags.map do |gitaly_tag| + if gitaly_tag.target_commit.present? + commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) + gitaly_commit = Gitlab::Git::Commit.new(commit) + end + + Gitlab::Git::Tag.new( + @repository, + encode!(gitaly_tag.name.dup), + gitaly_tag.id, + gitaly_commit, + encode!(gitaly_tag.message.chomp) + ) + end + end + end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name.dup, + author_email: response.commit_author.email.dup, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name.dup, + committer_email: response.commit_committer.email.dup + } + + Gitlab::Git::Commit.decorate(hash) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb new file mode 100644 index 00000000000..13e75b256a7 --- /dev/null +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -0,0 +1,32 @@ +module Gitlab + module GitalyClient + class RepositoryService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def exists? + request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :repository_service, :exists, request).exists + end + + def garbage_collect(create_bitmap) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request) + end + + def repack_full(create_bitmap) + request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :repack_full, request) + end + + def repack_incremental + request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 86d055d3533..f5a4c5493ef 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -4,7 +4,6 @@ module Gitlab class << self def repository(repository_storage, relative_path) Gitaly::Repository.new( - path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), storage_name: repository_storage, relative_path: relative_path ) diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index a8c0b47e786..266b1a6fece 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -254,7 +254,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb index 6c592ff469c..0396122eeb9 100644 --- a/lib/gitlab/github_import/wiki_formatter.rb +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -7,8 +7,8 @@ module Gitlab @project = project end - def path_with_namespace - "#{project.path_with_namespace}.wiki" + def disk_path + "#{project.disk_path}.wiki" end def import_url diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 319633656ff..2d1ae6a5925 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -2,11 +2,14 @@ module Gitlab module GonHelper + include WebpackHelper + def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb new file mode 100644 index 00000000000..e1d1724295a --- /dev/null +++ b/lib/gitlab/gpg.rb @@ -0,0 +1,62 @@ +module Gitlab + module Gpg + extend self + + module CurrentKeyChain + extend self + + def add(key) + GPGME::Key.import(key) + end + + def fingerprints_from_key(key) + import = GPGME::Key.import(key) + + return [] if import.imported == 0 + + import.imports.map(&:fingerprint) + end + end + + def fingerprints_from_key(key) + using_tmp_keychain do + CurrentKeyChain.fingerprints_from_key(key) + end + end + + def primary_keyids_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).map { |raw_key| raw_key.primary_subkey.keyid } + end + end + + def user_infos_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| + raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + end + end + end + + def using_tmp_keychain + Dir.mktmpdir do |dir| + @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] + @original_dirs.push(dir) + + GPGME::Engine.home_dir = dir + + return_value = yield + + @original_dirs.pop + + GPGME::Engine.home_dir = @original_dirs[-1] + + return_value + end + end + end +end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb new file mode 100644 index 00000000000..55428b85207 --- /dev/null +++ b/lib/gitlab/gpg/commit.rb @@ -0,0 +1,85 @@ +module Gitlab + module Gpg + class Commit + attr_reader :commit + + def initialize(commit) + @commit = commit + + @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + end + + def has_signature? + !!(@signature_text && @signed_text) + end + + def signature + return unless has_signature? + + cached_signature = GpgSignature.find_by(commit_sha: commit.sha) + return cached_signature if cached_signature.present? + + using_keychain do |gpg_key| + create_cached_signature!(gpg_key) + end + end + + def update_signature!(cached_signature) + using_keychain do |gpg_key| + cached_signature.update_attributes!(attributes(gpg_key)) + end + end + + private + + def using_keychain + Gitlab::Gpg.using_tmp_keychain do + # first we need to get the keyid from the signature to query the gpg + # key belonging to the keyid. + # This way we can add the key to the temporary keychain and extract + # the proper signature. + gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + + if gpg_key + Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) + @verified_signature = nil + end + + yield gpg_key + end + end + + def verified_signature + @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + break verified_signature + end + end + + def create_cached_signature!(gpg_key) + GpgSignature.create!(attributes(gpg_key)) + end + + def attributes(gpg_key) + user_infos = user_infos(gpg_key) + + { + commit_sha: commit.sha, + project: commit.project, + gpg_key: gpg_key, + gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_user_name: user_infos[:name], + gpg_key_user_email: user_infos[:email], + valid_signature: gpg_signature_valid_signature_value(gpg_key) + } + end + + def gpg_signature_valid_signature_value(gpg_key) + !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + end + + def user_infos(gpg_key) + gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} + end + end + end +end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb new file mode 100644 index 00000000000..3bb491120ba --- /dev/null +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -0,0 +1,19 @@ +module Gitlab + module Gpg + class InvalidGpgSignatureUpdater + def initialize(gpg_key) + @gpg_key = gpg_key + end + + def run + GpgSignature + .select(:id, :commit_sha, :project_id) + .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) + .find_each do |gpg_signature| + Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) + end + end + end + end +end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/group_hierarchy.rb index e9d5d52cabb..5a31e56cb30 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/group_hierarchy.rb @@ -3,33 +3,38 @@ module Gitlab # # This class uses recursive CTEs and as a result will only work on PostgreSQL. class GroupHierarchy - attr_reader :base, :model - - # base - An instance of ActiveRecord::Relation for which to get parent or - # child groups. - def initialize(base) - @base = base - @model = base.model + attr_reader :ancestors_base, :descendants_base, :model + + # ancestors_base - An instance of ActiveRecord::Relation for which to + # get parent groups. + # descendants_base - An instance of ActiveRecord::Relation for which to + # get child groups. If omitted, ancestors_base is used. + def initialize(ancestors_base, descendants_base = ancestors_base) + raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model + + @ancestors_base = ancestors_base + @descendants_base = descendants_base + @model = ancestors_base.model end - # Returns a relation that includes the base set of groups and all their - # ancestors (recursively). + # Returns a relation that includes the ancestors_base set of groups + # and all their ancestors (recursively). def base_and_ancestors - return model.none unless Group.supports_nested_groups? + return ancestors_base unless Group.supports_nested_groups? base_and_ancestors_cte.apply_to(model.all) end - # Returns a relation that includes the base set of groups and all their - # descendants (recursively). + # Returns a relation that includes the descendants_base set of groups + # and all their descendants (recursively). def base_and_descendants - return model.none unless Group.supports_nested_groups? + return descendants_base unless Group.supports_nested_groups? base_and_descendants_cte.apply_to(model.all) end - # Returns a relation that includes the base groups, their ancestors, and the - # descendants of the base groups. + # Returns a relation that includes the base groups, their ancestors, + # and the descendants of the base groups. # # The resulting query will roughly look like the following: # @@ -48,8 +53,10 @@ module Gitlab # # Using this approach allows us to further add criteria to the relation with # Rails thinking it's selecting data the usual way. + # + # If nested groups are not supported, ancestors_base is returned. def all_groups - return base unless Group.supports_nested_groups? + return ancestors_base unless Group.supports_nested_groups? ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -60,11 +67,11 @@ module Gitlab union = SQL::Union.new([model.unscoped.from(ancestors_table), model.unscoped.from(descendants_table)]) - model. - unscoped. - with. - recursive(ancestors.to_arel, descendants.to_arel). - from("(#{union.to_sql}) #{model.table_name}") + model + .unscoped + .with + .recursive(ancestors.to_arel, descendants.to_arel) + .from("(#{union.to_sql}) #{model.table_name}") end private @@ -72,13 +79,13 @@ module Gitlab def base_and_ancestors_cte cte = SQL::RecursiveCTE.new(:base_and_ancestors) - cte << base.except(:order) + cte << ancestors_base.except(:order) # Recursively get all the ancestors of the base set. - cte << model. - from([groups_table, cte.table]). - where(groups_table[:id].eq(cte.table[:parent_id])). - except(:order) + cte << model + .from([groups_table, cte.table]) + .where(groups_table[:id].eq(cte.table[:parent_id])) + .except(:order) cte end @@ -86,13 +93,13 @@ module Gitlab def base_and_descendants_cte cte = SQL::RecursiveCTE.new(:base_and_descendants) - cte << base.except(:order) + cte << descendants_base.except(:order) # Recursively get all the descendants of the base set. - cte << model. - from([groups_table, cte.table]). - where(groups_table[:parent_id].eq(cte.table[:id])). - except(:order) + cte << model + .from([groups_table, cte.table]) + .where(groups_table[:parent_id].eq(cte.table[:id])) + .except(:order) cte end diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7de6d4d9367..8b365dab185 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -27,10 +27,10 @@ module Gitlab Metric.new(name, value, labels) end - def with_timing(proc) + def with_timing start = Time.now - result = proc.call - yield result, Time.now.to_f - start.to_f + result = yield + [result, Time.now.to_f - start.to_f] end def catch_timeout(seconds, &block) diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index e78b7f22e03..9e91c135956 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,49 +10,47 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - tmp_file_path = tmp_file_path(storage_name) - if !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - elsif !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) else - HealthChecks::Result.new(true, nil, shard: storage_name) + with_temp_file(storage_name) do |tmp_file_path| + if !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + end end rescue RuntimeError => ex message = "unexpected error #{ex} when checking storage #{storage_name}" Rails.logger.error(message) HealthChecks::Result.new(false, message, shard: storage_name) - ensure - delete_test_file(tmp_file_path) end end end def metrics repository_storages.flat_map do |storage_name| - tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + storage_stat_metrics(storage_name), + storage_write_metrics(storage_name), + storage_read_metrics(storage_name) ].flatten end end private - def operation_metrics(ok_metric, latency_metric, operation, **labels) - with_timing operation do |result, elapsed| - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - end + def operation_metrics(ok_metric, latency_metric, **labels) + result, elapsed = yield + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] rescue RuntimeError => ex - Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] end @@ -68,19 +66,36 @@ module Gitlab Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end - def tmp_file_path(storage_name) - Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + def with_temp_file(storage_name) + temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } + yield temp_file_path + ensure + delete_test_file(temp_file_path) end - def path(storage_name) + def storage_path(storage_name) storages_paths&.dig(storage_name, 'path') end + # All below test methods use shell commands to perform actions on storage volumes. + # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, + # we can rely on shell commands to be terminated once `timeout` kills them. + # + # However we also fallback to pure Ruby file operations in case a specific shell command is missing + # so we are still able to perform healthchecks and gather metrics from such system. + + def delete_test_file(tmp_path) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) + status.zero? + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + def storage_stat_test(storage_name) - stat_path = File.join(path(storage_name), '.') + stat_path = File.join(storage_path(storage_name), '.') begin _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status == 0 + status.zero? rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? end @@ -90,7 +105,7 @@ module Gitlab _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT written_bytes == RANDOM_STRING.length @@ -100,17 +115,33 @@ module Gitlab _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT file_contents = File.read(tmp_path) rescue Errno::ENOENT file_contents == RANDOM_STRING end - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status == 0 - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT + def storage_stat_metrics(storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do + with_timing { storage_stat_test(storage_name) } + end + end + + def storage_write_metrics(storage_name) + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + with_timing { storage_write_test(tmp_file_path) } + end + end + end + + def storage_read_metrics(storage_name) + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + storage_write_test(tmp_file_path) # writes data used by read test + with_timing { storage_read_test(tmp_file_path) } + end + end end end end diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb new file mode 100644 index 00000000000..b3c759b4730 --- /dev/null +++ b/lib/gitlab/health_checks/prometheus_text_format.rb @@ -0,0 +1,40 @@ +module Gitlab + module HealthChecks + class PrometheusTextFormat + def marshal(metrics) + "#{metrics_with_type_declarations(metrics).join("\n")}\n" + end + + private + + def metrics_with_type_declarations(metrics) + type_declaration_added = {} + + metrics.flat_map do |metric| + metric_lines = [] + + unless type_declaration_added.key?(metric.name) + type_declaration_added[metric.name] = true + metric_lines << metric_type_declaration(metric) + end + + metric_lines << metric_text(metric) + end + end + + def metric_type_declaration(metric) + "# TYPE #{metric.name} gauge" + end + + def metric_text(metric) + labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || '' + + if labels.empty? + "#{metric.name} #{metric.value}" + else + "#{metric.name}{#{labels}} #{metric.value}" + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..f5026171ba4 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -15,14 +15,13 @@ module Gitlab end def metrics - with_timing method(:check) do |result, elapsed| - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) - [ - metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) - ] - end + result, elapsed = with_timing(&method(:check)) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency_seconds", elapsed) + ] end private diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 83bc230df3e..5408a1a6838 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,16 +1,8 @@ module Gitlab class Highlight def self.highlight(blob_name, blob_content, repository: nil, plain: false) - new(blob_name, blob_content, repository: repository). - highlight(blob_content, continue: false, plain: plain) - end - - def self.highlight_lines(repository, ref, file_name) - blob = repository.blob_at(ref, file_name) - return [] unless blob - - blob.load_all_data!(repository) - highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) + new(blob_name, blob_content, repository: repository) + .highlight(blob_content, continue: false, plain: plain) end attr_reader :blob_name diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f7ac48f7dbd..cc282d1415b 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -6,9 +6,17 @@ module Gitlab 'en' => 'English', 'es' => 'Español', 'de' => 'Deutsch', + 'fr' => 'Français', + 'pt_BR' => 'Português (Brasil)', 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文(香港)', - 'zh_TW' => '繁體中文(臺灣)' + 'zh_HK' => '繁體中文 (香港)', + 'zh_TW' => '繁體中文 (臺灣)', + 'bg' => 'български', + 'ru' => 'Русский', + 'eo' => 'Esperanto', + 'it' => 'Italiano', + 'uk' => 'Українська', + 'ja' => '日本語' }.freeze def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 27d5a9198b6..3470a09eaf0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.7'.freeze + VERSION = '0.1.8'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d0f3cf2b514..c8ad3a7a5e0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -26,7 +26,9 @@ project_tree: - notes: - :author - :events - - :merge_request_diff + - merge_request_diff: + - :merge_request_diff_commits + - :merge_request_diff_files - :events - :timelogs - label_links: @@ -38,6 +40,7 @@ project_tree: - notes: - :author - :events + - :stages - :statuses - :triggers - :pipeline_schedules @@ -91,10 +94,13 @@ excluded_attributes: - :expired_at merge_request_diff: - :st_diffs + merge_request_diff_files: + - :diff issues: - :milestone_id merge_requests: - :milestone_id + - :ref_fetched award_emoji: - :awardable_id statuses: @@ -112,6 +118,8 @@ methods: - :type merge_request_diff: - :utf8_st_diffs + merge_request_diff_files: + - :utf8_diff merge_requests: - :diff_head_sha project: diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 48c09dafcb6..b48f63bcd7e 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -83,7 +83,9 @@ module Gitlab # +value+ existing model to be included in the hash # +json_config_hash+ the original hash containing the root model def add_model_value(current_key, value, json_config_hash) - @attributes_finder.parse(value) { |hash| value = { value => hash } } + @attributes_finder.parse(value) do |hash| + value = { value => hash } unless value.is_a?(Hash) + end add_to_array(current_key, json_config_hash, value) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e23a4715f..20580459046 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,6 +3,7 @@ module Gitlab class RelationFactory OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', @@ -70,6 +71,7 @@ module Gitlab @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diff_commits if @relation_name == :merge_request_diff + set_diff if @relation_name == :merge_request_diff_files end def update_user_references @@ -201,6 +203,10 @@ module Gitlab HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits']) end + def set_diff + @relation_hash['diff'] = @relation_hash.delete('utf8_diff') + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index c824d3ea9fc..32ca2809b2f 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle) + gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 62a2553675c..f9ae5079d7c 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -24,6 +24,7 @@ module Gitlab end def uploads_path + # TODO: decide what to do with uploads. We will use UUIDs here too? File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) end end diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 8db91d25a4b..208f0e1bbea 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -14,7 +14,7 @@ module Gitlab # timeout - The maximum amount of seconds to block the caller for. This # ensures we don't indefinitely block a caller in case a job takes # long to process, or is never processed. - def wait(timeout = 60) + def wait(timeout = 10) start = Time.current while (Time.current - start) <= timeout diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 4a6091488c8..cdbdfa10d0e 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -8,13 +8,13 @@ module Gitlab ) # Filters an array of pods (as returned by the kubernetes API) by their labels - def filter_pods(pods, labels = {}) - pods.select do |pod| - metadata = pod.fetch("metadata", {}) - pod_labels = metadata.fetch("labels", nil) - next unless pod_labels + def filter_by_label(items, labels = {}) + items.select do |item| + metadata = item.fetch("metadata", {}) + item_labels = metadata.fetch("labels", nil) + next unless item_labels - labels.all? { |k, v| pod_labels[k.to_s] == v } + labels.all? { |k, v| item_labels[k.to_s] == v } end end @@ -76,5 +76,44 @@ module Gitlab url.to_s end + + def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + config = { + apiVersion: 'v1', + clusters: [ + name: 'gitlab-deploy', + cluster: { + server: url + } + ], + contexts: [ + name: 'gitlab-deploy', + context: { + cluster: 'gitlab-deploy', + namespace: namespace, + user: 'gitlab-deploy' + } + ], + 'current-context': 'gitlab-deploy', + kind: 'Config', + users: [ + { + name: 'gitlab-deploy', + user: { token: token } + } + ] + } + + kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem + + config.deep_stringify_keys + end + + private + + def kubeconfig_embed_ca_pem(config, ca_pem) + cluster = config.dig(:clusters, 0, :cluster) + cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 54a5b1d31cd..fb68627dedf 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,8 +16,8 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - user.last_credential_check_at = Time.now - user.save + Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + true else false diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 7b05290e5cc..8867a91c244 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -101,7 +101,7 @@ module Gitlab end def user_attributes - %W(#{config.uid} cn mail dn) + %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] end end end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index 4745311402c..ed1de73f8c6 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -42,7 +42,7 @@ module Gitlab end def adapter - OmniAuth::LDAP::Adaptor.new(config.options.symbolize_keys) + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) end def config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 6fdf68641e2..c8f19cd52d5 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -2,6 +2,12 @@ module Gitlab module LDAP class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + attr_accessor :provider, :options def self.enabled? @@ -12,6 +18,12 @@ module Gitlab Gitlab.config.ldap.servers.values end + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + def self.providers servers.map { |server| server['provider_name'] } end @@ -39,7 +51,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption + encryption: encryption_options ) opts.merge!(auth_options) if has_auth? @@ -50,9 +62,10 @@ module Gitlab def omniauth_options opts = base_options.merge( base: base, - method: options['method'], + encryption: options['encryption'], filter: omniauth_user_filter, - name_proc: name_proc + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] ) if has_auth? @@ -62,6 +75,9 @@ module Gitlab ) end + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts end @@ -157,15 +173,37 @@ module Gitlab base_config.servers.values.find { |server| server['provider_name'] == provider } end - def encryption - case options['method'].to_s - when 'ssl' - :simple_tls - when 'tls' - :start_tls - else - nil - end + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts end def auth_options diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 2d5e47a6f3b..39180dc17d9 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -10,9 +10,9 @@ module Gitlab class << self def find_by_uid_and_provider(uid, provider) # LDAP distinguished name is case-insensitive - identity = ::Identity. - where(provider: provider). - iwhere(extern_uid: uid).last + identity = ::Identity + .where(provider: provider) + .iwhere(extern_uid: uid).last identity && identity.user end end @@ -41,11 +41,6 @@ module Gitlab def update_user_attributes if persisted? - if auth_hash.has_email? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) @@ -55,10 +50,6 @@ module Gitlab # For an existing identity with no change in DN, this line changes nothing. identity.extern_uid = auth_hash.uid end - - gl_user.ldap_email = auth_hash.has_email? - - gl_user end def changed? @@ -69,6 +60,10 @@ module Gitlab ldap_config.block_auto_created_users end + def sync_email_from_provider? + true + end + def allowed? Gitlab::LDAP::Access.allowed?(gl_user) end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cb8db2f1e9f..4779755bb22 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,161 +1,10 @@ module Gitlab module Metrics - extend Gitlab::CurrentSettings - - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def self.settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + extend Gitlab::Metrics::InfluxDb + extend Gitlab::Metrics::Prometheus def self.enabled? - settings[:enabled] || false - end - - def self.mri? - RUBY_ENGINE == 'ruby' - end - - def self.method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - @method_call_threshold ||= settings[:method_call_threshold] - end - - def self.pool - @pool - end - - def self.submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError - end - end - end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - - def self.prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def self.escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def self.measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) * 1000.0 - cpu_time = cpu_stop - cpu_start - - trans.increment("#{name}_real_time", real_time) - trans.increment("#{name}_cpu_time", cpu_time) - trans.increment("#{name}_call_count", 1) - - retval - end - - # Adds a tag to the current transaction (if any) - # - # name - The name of the tag to add. - # value - The value of the tag. - def self.tag_transaction(name, value) - trans = current_transaction - - trans&.add_tag(name, value) - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def self.action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def self.add_event(*args) - trans = current_transaction - - trans&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def self.series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - if enabled? - @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client. - new(udp: { host: host, port: port }) - end + influx_metrics_enabled? || prometheus_metrics_enabled? end end end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb new file mode 100644 index 00000000000..219accfc029 --- /dev/null +++ b/lib/gitlab/metrics/base_sampler.rb @@ -0,0 +1,94 @@ +require 'logger' +module Gitlab + module Metrics + class BaseSampler + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance + end + + attr_reader :running + + # interval - The sampling interval in seconds. + def initialize(interval) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return if running + @running = true + + @thread = Thread.new do + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + end + end + + def stop + @mutex.synchronize do + return unless running + + @running = false + + if @thread + @thread.wakeup if @thread.alive? + @thread.join + @thread = nil + end + end + end + + def safe_sample + sample + rescue => e + Rails.logger.warn("#{self.class}: #{e}, stopping") + stop + end + + def sample + raise NotImplementedError + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb new file mode 100644 index 00000000000..d7c56463aac --- /dev/null +++ b/lib/gitlab/metrics/influx_db.rb @@ -0,0 +1,170 @@ +module Gitlab + module Metrics + module InfluxDb + extend Gitlab::CurrentSettings + extend self + + MUTEX = Mutex.new + private_constant :MUTEX + + def influx_metrics_enabled? + settings[:enabled] || false + end + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 + } + end + + def mri? + RUBY_ENGINE == 'ruby' + end + + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= settings[:method_call_threshold] + end + + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end + end + end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) + end + + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def escape_value(value) + value.to_s.gsub('=', '\\=') + end + + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def tag_transaction(name, value) + trans = current_transaction + + trans&.add_tag(name, value) + end + + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction + + trans&.action = action + end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction + + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client + .new(udp: { host: host, port: port }) + end + end + end + @pool + end + end + end + end +end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/influx_sampler.rb index 0000450d9bb..6db1dd755b7 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/influx_sampler.rb @@ -5,14 +5,11 @@ module Gitlab # This class is used to gather statistics that can't be directly associated # with a transaction such as system memory usage, garbage collection # statistics, etc. - class Sampler + class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - interval_half = interval.to_f / 2 - - @interval = interval - @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @last_step = nil + super(interval) + @last_step = nil @metrics = [] @@ -26,18 +23,6 @@ module Gitlab end end - def start - Thread.new do - Thread.current.abort_on_exception = true - - loop do - sleep(sleep_interval) - - sample - end - end - end - def sample sample_memory_usage sample_file_descriptors @@ -86,7 +71,7 @@ module Gitlab end def sample_gc - time = GC::Profiler.total_time * 1000.0 + time = GC::Profiler.total_time * 1000.0 stats = GC.stat.merge(total_time: time) # We want the difference of GC runs compared to the last sample, not the @@ -111,23 +96,6 @@ module Gitlab def sidekiq? Sidekiq.server? end - - # Returns the sleep interval with a random adjustment. - # - # The random adjustment is put in place to ensure we: - # - # 1. Don't generate samples at the exact same interval every time (thus - # potentially missing anything that happens in between samples). - # 2. Don't sample data at the same interval two times in a row. - def sleep_interval - while step = @interval_steps.sample - if step != @last_step - @last_step = step - - return @interval + @last_step - end - end - end end end end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb new file mode 100644 index 00000000000..3b5a2907195 --- /dev/null +++ b/lib/gitlab/metrics/null_metric.rb @@ -0,0 +1,10 @@ +module Gitlab + module Metrics + # Mocks ::Prometheus::Client::Metric and all derived metrics + class NullMetric + def method_missing(name, *args, &block) + nil + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb new file mode 100644 index 00000000000..460dab47276 --- /dev/null +++ b/lib/gitlab/metrics/prometheus.rb @@ -0,0 +1,57 @@ +require 'prometheus/client' + +module Gitlab + module Metrics + module Prometheus + include Gitlab::CurrentSettings + + def metrics_folder_present? + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) + end + + def prometheus_metrics_enabled? + return @prometheus_metrics_enabled if defined?(@prometheus_metrics_enabled) + + @prometheus_metrics_enabled = prometheus_metrics_enabled_unmemoized + end + + def registry + @registry ||= ::Prometheus::Client.registry + end + + def counter(name, docstring, base_labels = {}) + provide_metric(name) || registry.counter(name, docstring, base_labels) + end + + def summary(name, docstring, base_labels = {}) + provide_metric(name) || registry.summary(name, docstring, base_labels) + end + + def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) + provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode) + end + + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets) + end + + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.new + end + end + + private + + def prometheus_metrics_enabled_unmemoized + metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false + end + end + end +end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 3aaebb3e9c3..aba3e0df382 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,13 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process. - clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process + .clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) end else def self.cpu_time - Process. - clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process + .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) end end diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb new file mode 100644 index 00000000000..f6987252039 --- /dev/null +++ b/lib/gitlab/metrics/unicorn_sampler.rb @@ -0,0 +1,48 @@ +module Gitlab + module Metrics + class UnicornSampler < BaseSampler + def initialize(interval) + super(interval) + end + + def unicorn_active_connections + @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + end + + def unicorn_queued_connections + @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + end + + def enabled? + # Raindrops::Linux.tcp_listener_stats is only present on Linux + unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) + end + + def sample + Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + end + + Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + end + end + + private + + def tcp_listeners + @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) + end + + def unix_listeners + @unix_listeners ||= Unicorn.listener_names - tcp_listeners + end + + def unicorn_with_listeners? + defined?(Unicorn) && Unicorn.listener_names.any? + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index afd24b4dcc5..e8330917e91 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,6 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash + update_email end def persisted? @@ -31,7 +32,7 @@ module Gitlab block_after_save = needs_blocking? - gl_user.save! + Users::UpdateService.new(gl_user).execute! gl_user.block if block_after_save @@ -100,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end @@ -161,12 +166,17 @@ module Gitlab username ||= auth_hash.username email ||= auth_hash.email + valid_username = ::Namespace.clean_path(username) + + uniquify = Uniquify.new + valid_username = uniquify.string(valid_username) { |s| !DynamicPathValidator.valid_user_path?(s) } + name = auth_hash.name - name = ::Namespace.clean_path(username) if name.strip.empty? + name = valid_username if name.strip.empty? { name: name, - username: ::Namespace.clean_path(username), + username: valid_username, email: email, password: auth_hash.password, password_confirmation: auth_hash.password, @@ -174,6 +184,22 @@ module Gitlab } end + def sync_email_from_provider? + auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + end + + def update_email + if auth_hash.has_email? && sync_email_from_provider? + if persisted? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end + + gl_user.external_email = true + gl_user.email_provider = auth_hash.provider + end + end + def log Gitlab::AppLogger end diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 31a24460f0f..fc3f21233dd 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -6,8 +6,8 @@ module Gitlab # input - the source text in a markup format # def self.render(file_name, input, context) - html = GitHub::Markup.render(file_name, input). - force_encoding(input.encoding) + html = GitHub::Markup.render(file_name, input) + .force_encoding(input.encoding) context[:pipeline] = :markup html = Banzai.render(html, context) diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 0d541935bc6..22332474945 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -34,7 +34,7 @@ module Gitlab write_csv do |csv| ActiveRecord::Base.transaction do - User.with_two_factor.in_batches do |relation| + User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) rows.each do |row| user = %i[id ciphertext iv salt].zip(row).to_h diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 9ff6829cd49..894bd5efae5 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -14,42 +14,42 @@ module Gitlab TOP_LEVEL_ROUTES = %w[ - .well-known + 404.html + 422.html + 500.html + 502.html + 503.html abuse_reports admin - all api + apple-touch-icon-precomposed.png + apple-touch-icon.png assets autocomplete ci dashboard + deploy.html explore + favicon.ico files groups health_check help - hooks import invites - issues jwt koding - member - merge_requests - new - notes notification_settings oauth profile projects public - repository robots.txt s search sent_notifications - services + slash-command-logo.png snippets - teams u unicorn_test unsubscribes @@ -111,6 +111,7 @@ module Gitlab # this group would not be accessible through `/groups/parent/activity` since # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ + - activity analytics audit_events diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb new file mode 100644 index 00000000000..56112ec2301 --- /dev/null +++ b/lib/gitlab/performance_bar.rb @@ -0,0 +1,34 @@ +module Gitlab + module PerformanceBar + include Gitlab::CurrentSettings + + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes + + def self.enabled?(user = nil) + return false unless user && allowed_group_id + + allowed_user_ids.include?(user.id) + end + + def self.allowed_group_id + current_application_settings.performance_bar_allowed_group_id + end + + def self.allowed_user_ids + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do + group = Group.find_by_id(allowed_group_id) + + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end + end + end + + def self.expire_allowed_user_ids_cache + Rails.cache.delete(ALLOWED_USER_IDS_KEY) + end + end +end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb new file mode 100644 index 00000000000..67fee8c227d --- /dev/null +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -0,0 +1,46 @@ +# Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb +# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb +module Gitlab + module PerformanceBar + module PeekQueryTracker + def sorted_queries + PEEK_DB_CLIENT.query_details + .sort { |a, b| b[:duration] <=> a[:duration] } + end + + def results + super.merge(queries: sorted_queries) + end + + private + + def setup_subscribers + super + + # Reset each counter when a new request starts + before_request do + PEEK_DB_CLIENT.query_details = [] + end + + subscribe('sql.active_record') do |_, start, finish, _, data| + if RequestStore.active? && RequestStore.store[:peek_enabled] + # data[:cached] is only available starting from Rails 5.1.0 + # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 + # Before that, data[:name] was set to 'CACHE' + # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end + end + end + end + + def track_query(raw_query, bindings, start, finish) + query = Gitlab::Sherlock::Query.new(raw_query, start, finish) + query_info = { duration: query.duration.round(3), sql: query.formatted_query } + + PEEK_DB_CLIENT.query_details << query_info + end + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index bb0df1e3dad..15b8beacf60 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -28,34 +28,34 @@ module Gitlab # Projects that belong directly to any of the groups the user has # access to. - Namespace. - unscoped. - select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]). - from(cte_alias). - joins(:projects), + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), # Projects shared with any of the namespaces the user has access to. - Namespace. - unscoped. - select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]). - from(cte_alias). - joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id'). - joins('INNER JOIN projects ON projects.id = project_group_links.project_id'). - joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id'). - where('p_ns.share_with_group_lock IS FALSE') + Namespace + .unscoped + .select([links[:project_id], + least(cte_alias[:access_level], + links[:group_access], + 'access_level')]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') ] union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization. - unscoped. - with. - recursive(cte.to_arel). - select_from_union(union) + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(union) end private @@ -68,17 +68,17 @@ module Gitlab namespaces = Namespace.arel_table # Namespaces the user is a member of. - cte << user.groups. - select([namespaces[:id], members[:access_level]]). - except(:order) + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) # Sub groups of any groups the user is a member of. cte << Group.select([namespaces[:id], greatest(members[:access_level], - cte.table[:access_level], 'access_level')]). - joins(join_cte(cte)). - joins(join_members). - except(:order) + cte.table[:access_level], 'access_level')]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) cte end @@ -88,11 +88,11 @@ module Gitlab members = Member.arel_table namespaces = Namespace.arel_table - cond = members[:source_id]. - eq(namespaces[:id]). - and(members[:source_type].eq('Namespace')). - and(members[:requested_at].eq(nil)). - and(members[:user_id].eq(user.id)) + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index 627e8c5fba2..ad87540e6c2 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -26,9 +26,9 @@ module Gitlab union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization. - unscoped. - select_from_union(union) + ProjectAuthorization + .unscoped + .select_from_union(union) end end end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb new file mode 100644 index 00000000000..cb95daf2260 --- /dev/null +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Prometheus + module AdditionalMetricsParser + extend self + + def load_groups_from_yaml + additional_metrics_raw.map(&method(:group_from_entry)) + end + + private + + def validate!(obj) + raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid? + end + + def group_from_entry(entry) + entry[:name] = entry.delete(:group) + entry[:metrics]&.map! do |entry| + Metric.new(entry).tap(&method(:validate!)) + end + + MetricGroup.new(entry).tap(&method(:validate!)) + end + + def additional_metrics_raw + load_yaml_file&.map(&:deep_symbolize_keys).freeze + end + + def load_yaml_file + @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + end + end + end +end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb new file mode 100644 index 00000000000..f54b2c6aaff --- /dev/null +++ b/lib/gitlab/prometheus/metric.rb @@ -0,0 +1,16 @@ +module Gitlab + module Prometheus + class Metric + include ActiveModel::Model + + attr_accessor :title, :required_metrics, :weight, :y_label, :queries + + validates :title, :required_metrics, :weight, :y_label, :queries, presence: true + + def initialize(params = {}) + super(params) + @y_label ||= 'Values' + end + end + end +end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb new file mode 100644 index 00000000000..729fef34b35 --- /dev/null +++ b/lib/gitlab/prometheus/metric_group.rb @@ -0,0 +1,14 @@ +module Gitlab + module Prometheus + class MetricGroup + include ActiveModel::Model + + attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true + + def self.all + AdditionalMetricsParser.load_groups_from_yaml + end + end + end +end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb new file mode 100644 index 00000000000..49cc0e16080 --- /dev/null +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -0,0 +1,5 @@ +module Gitlab + module Prometheus + ParsingError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb new file mode 100644 index 00000000000..69d055c901c --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -0,0 +1,21 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsDeploymentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + query_metrics( + common_query_context( + deployment.environment, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + ) + ) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb new file mode 100644 index 00000000000..db4708b22e4 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -0,0 +1,17 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsEnvironmentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + query_metrics( + common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) + ) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 2a2eb4ae57f..c60828165bd 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client - delegate :query_range, :query, to: :client, prefix: true + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 2cc08731f8d..170f483540e 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,26 +1,31 @@ -module Gitlab::Prometheus::Queries - class DeploymentQuery < BaseQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - environment_slug = deployment.environment.slug +module Gitlab + module Prometheus + module Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + environment_slug = deployment.environment.slug - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 01d756d7284..66f29d95177 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,20 +1,25 @@ -module Gitlab::Prometheus::Queries - class EnvironmentQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f +module Gitlab + module Prometheus + module Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb new file mode 100644 index 00000000000..d4894c87f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -0,0 +1,80 @@ +module Gitlab + module Prometheus + module Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze + + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end + + private + + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) + + groups = {} + + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end + + groups + end + + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now + + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end + + def has_matching_label(series_info) + series_info.key?('environment') + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end + + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end + + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb new file mode 100644 index 00000000000..7ac6162b54d --- /dev/null +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -0,0 +1,86 @@ +module Gitlab + module Prometheus + module Queries + module QueryAdditionalMetrics + def query_metrics(query_context) + query_processor = method(:process_query).curry[query_context] + + groups = matched_metrics.map do |group| + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + y_label: metric.y_label, + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) + } + end + + { + group: group.name, + priority: group.priority, + metrics: metrics.select(&method(:metric_with_any_queries)) + } + end + + groups.select(&method(:group_with_any_metrics)) + end + + private + + def metric_with_any_queries(metric) + metric[:queries]&.count&.> 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count&.> 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_query(context, query) + query = query.dup + result = + if query.key?(:query_range) + query[:query_range] %= context + client_query_range(query[:query_range], start: context[:timeframe_start], stop: context[:timeframe_end]) + else + query[:query] %= context + client_query(query[:query], time: context[:timeframe_end]) + end + + query[:result] = result&.map(&:deep_symbolize_keys) + query + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def matched_metrics + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + group + end + + result.select { |group| group.metrics.any? } + end + + def common_query_context(environment, timeframe_start:, timeframe_end:) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end, + ci_environment_slug: environment.slug, + kube_namespace: environment.project.kubernetes_service&.actual_namespace || '', + environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + } + end + end + end + end +end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 5b51a1779dd..aa94614bf18 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -29,6 +29,14 @@ module Gitlab end end + def label_values(name = '__name__') + json_api_get("label/#{name}/values") + end + + def series(*matches, start: 8.hours.ago, stop: Time.now) + json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) + end + private def json_api_get(type, args = {}) diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 12a385f90fd..3937d9c153a 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -1,5 +1,5 @@ module Gitlab - module SlashCommands + module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :params, :condition_block, :parse_params_block, :action_block @@ -48,17 +48,23 @@ module Gitlab end def to_h(opts) + context = OpenStruct.new(opts) + desc = description if desc.respond_to?(:call) - context = OpenStruct.new(opts) desc = context.instance_exec(&desc) rescue '' end + prms = params + if prms.respond_to?(:call) + prms = Array(context.instance_exec(&prms)) rescue params + end + { name: name, aliases: aliases, description: desc, - params: params + params: prms } end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index 614bafbe1b2..536765305e1 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -1,5 +1,5 @@ module Gitlab - module SlashCommands + module QuickActions module Dsl extend ActiveSupport::Concern @@ -14,7 +14,7 @@ module Gitlab end class_methods do - # Allows to give a description to the next slash command. + # Allows to give a description to the next quick action. # This description is shown in the autocomplete menu. # It accepts a block that will be evaluated with the context given to # `CommandDefintion#to_h`. @@ -31,7 +31,7 @@ module Gitlab @description = block_given? ? block : text end - # Allows to define params for the next slash command. + # Allows to define params for the next quick action. # These params are shown in the autocomplete menu. # # Example: @@ -40,8 +40,8 @@ module Gitlab # command :command_key do |arguments| # # Awesome code block # end - def params(*params) - @params = params + def params(*params, &block) + @params = block_given? ? block : params end # Allows to give an explanation of what the command will do when @@ -105,9 +105,32 @@ module Gitlab # # Awesome code block # end def command(*command_names, &block) + define_command(CommandDefinition, *command_names, &block) + end + + # Registers a new substitution which is recognizable from body of email or + # comment. + # It accepts aliases and takes a block with the formatted content. + # + # Example: + # + # command :my_substitution, :alias_for_my_substitution do |text| + # "#{text} MY AWESOME SUBSTITUTION" + # end + def substitution(*substitution_names, &block) + define_command(SubstitutionDefinition, *substitution_names, &block) + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] + end + + private + + def define_command(klass, *command_names, &block) name, *aliases = command_names - definition = CommandDefinition.new( + definition = klass.new( name, aliases: aliases, description: @description, @@ -130,10 +153,6 @@ module Gitlab @condition_block = nil @parse_params_block = nil end - - def definition_by_name(name) - command_definitions_by_name[name.to_sym] - end end end end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 6dbb467d70d..3ebfa3bd4b8 100644 --- a/lib/gitlab/slash_commands/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -1,10 +1,10 @@ module Gitlab - module SlashCommands + module QuickActions # This class takes an array of commands that should be extracted from a # given text. # # ``` - # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # ``` class Extractor attr_reader :command_definitions @@ -24,7 +24,7 @@ module Gitlab # # Usage: # ``` - # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" @@ -46,6 +46,8 @@ module Gitlab end end + content, commands = perform_substitutions(content, commands) + [content.strip, commands] end @@ -110,6 +112,26 @@ module Gitlab }mx end + def perform_substitutions(content, commands) + return unless content + + substitution_definitions = self.command_definitions.select do |definition| + definition.is_a?(Gitlab::QuickActions::SubstitutionDefinition) + end + + substitution_definitions.each do |substitution| + match_data = substitution.match(content) + if match_data + command = [substitution.name.to_s] + command << match_data[1] unless match_data[1].empty? + commands << command + end + content = substitution.perform_substitution(self, content) + end + + [content, commands] + end + def command_names(opts) command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb new file mode 100644 index 00000000000..032c49ed159 --- /dev/null +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -0,0 +1,24 @@ +module Gitlab + module QuickActions + class SubstitutionDefinition < CommandDefinition + # noop?=>true means these won't get extracted or removed by Gitlab::QuickActions::Extractor#extract_commands + # QuickActions::InterpretService#perform_substitutions handles them separately + def noop? + true + end + + def match(content) + content.match %r{^/#{all_names.join('|')} ?(.*)$} + end + + def perform_substitution(context, content) + return unless content + + all_names.each do |a_name| + content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + end + content + end + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 7668ecacc4b..f5b757ace77 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -33,7 +33,12 @@ module Gitlab def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue) + if project.issues_enabled? + @references[:all_issues] ||= references(:external_issue) + references(:issue) + else + @references[:external_issue] ||= references(:external_issue) + + references(:issue).select { |i| i.project_id != project.id } + end else @references[:issue] ||= references(:issue) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e4d2a992470..1adc5ec952a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,31 +19,33 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze - end - - def file_name_regex_message - "can contain only letters, digits, '_', '-', '@', '+' and '.'." - end - - def container_registry_reference_regex - Gitlab::PathRegex.git_reference_regex - end - ## - # Docker Distribution Registry 2.4.1 repository name rules + # Docker Distribution Registry repository / tag name rules + # + # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} end + ## + # We do not use regexp anchors here because these are not allowed when + # used as a routing constraint. + # + def container_registry_tag_regex + @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/ + end + + def environment_name_regex_chars + 'a-zA-Z0-9_/\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces" end def kubernetes_namespace_regex diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 878e03f61d7..3591fa9145e 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -3,16 +3,18 @@ module Gitlab NotFoundError = Class.new(StandardError) def self.parse(repo_path) + wiki = false project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) - project = Project.find_by_full_path(project_path) - if project_path.end_with?('.wiki') && !project - project = Project.find_by_full_path(project_path.chomp('.wiki')) + project, was_redirected = find_project(project_path) + + if project_path.end_with?('.wiki') && project.nil? + project, was_redirected = find_project(project_path.chomp('.wiki')) wiki = true - else - wiki = false end - [project, wiki] + redirected_path = project_path if was_redirected + + [project, wiki, redirected_path] end def self.strip_storage_path(repo_path, fail_on_not_found: true) @@ -30,5 +32,12 @@ module Gitlab result.sub(/\A\/*/, '') end + + def self.find_project(project_path) + project = Project.find_by_full_path(project_path, follow_redirects: true) + was_redirected = project && project.full_path.casecmp(project_path) != 0 + + [project, was_redirected] + end end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb new file mode 100644 index 00000000000..ccfe0d6bed3 --- /dev/null +++ b/lib/gitlab/request_forgery_protection.rb @@ -0,0 +1,39 @@ +# A module to check CSRF tokens in requests. +# It's used in API helpers and OmniAuth. +# Usage: GitLab::RequestForgeryProtection.call(env) + +module Gitlab + module RequestForgeryProtection + class Controller < ActionController::Base + protect_from_forgery with: :exception + + rescue_from ActionController::InvalidAuthenticityToken do |e| + logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" + logger.warn "Unlike the logs may suggest, this does not result in an actual 422 response to the user" + logger.warn "For API requests, the only effect is that `current_user` will be `nil` for the duration of the request" + + raise e + end + + def index + head :ok + end + end + + def self.app + @app ||= Controller.action(:index) + end + + def self.call(env) + app.call(env) + end + + def self.verified?(env) + call(env) + + true + rescue ActionController::InvalidAuthenticityToken + false + end + end +end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 877aa6e6a28..f3952657983 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -18,7 +18,11 @@ module Gitlab mapping = @map.find { |mapping| mapping[:source] === path } return unless mapping - path.sub(mapping[:source], mapping[:public]) + if mapping[:source].is_a?(String) + path.sub(mapping[:source], mapping[:public]) + else + mapping[:source].replace(path, mapping[:public]) + end end private @@ -35,7 +39,7 @@ module Gitlab source_pattern = source_pattern[1...-1].gsub('\/', '/') begin - source_pattern = /\A#{source_pattern}\z/ + source_pattern = Gitlab::UntrustedRegexp.new('\A' + source_pattern + '\z') rescue RegexpError => e raise FormatError, "Route map entry source is not a valid regular expression: #{e}" end diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb deleted file mode 100644 index 36d1a8a6f64..00000000000 --- a/lib/gitlab/routes/legacy_builds.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module Routes - class LegacyBuilds - def initialize(map) - @map = map - end - - def draw - @map.instance_eval do - resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do - collection do - resources :artifacts, only: [], controller: 'build_artifacts' do - collection do - get :latest_succeeded, - path: '*ref_name_and_path', - format: false - end - end - end - - member do - get :raw - end - - resource :artifacts, only: [], controller: 'build_artifacts' do - get :download - get :browse, path: 'browse(/*path)', format: false - get :file, path: 'file/*path', format: false - get :raw, path: 'raw/*path', format: false - end - end - end - end - end - end -end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 632e2d87500..e57890f1143 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -2,10 +2,35 @@ module Gitlab module Routing extend ActiveSupport::Concern + mattr_accessor :_includers + self._includers = [] + included do + Gitlab::Routing.includes_helpers(self) + include Gitlab::Routing.url_helpers end + def self.includes_helpers(klass) + self._includers << klass + end + + def self.add_helpers(mod) + url_helpers.include mod + url_helpers.extend mod + + GitlabRoutingHelper.include mod + GitlabRoutingHelper.extend mod + + app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module + app_url_helpers.include mod + app_url_helpers.extend mod + + _includers.each do |klass| + klass.include mod + end + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index b1d6ea665b7..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,7 +1,12 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab class Shell + GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze + Error = Class.new(StandardError) KeyAdder = Struct.new(:io) do @@ -30,8 +35,8 @@ module Gitlab end def version_required - @version_required ||= File.read(Rails.root. - join('GITLAB_SHELL_VERSION')).strip + @version_required ||= File.read(Rails.root + .join('GITLAB_SHELL_VERSION')).strip end def strip_key(key) @@ -66,9 +71,10 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'add-project', storage, "#{name}.git"]) end # Import repository @@ -79,13 +85,13 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. - output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) - raise Error, output unless status.zero? - true + cmd = [gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] + gitlab_shell_fast_execute_raise_error(cmd) end # Fetch remote for repository @@ -98,14 +104,13 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced args << '--no-tags' if no_tags - output, status = Popen.popen(args) - raise Error, output unless status.zero? - true + gitlab_shell_fast_execute_raise_error(args) end # Move repository @@ -116,9 +121,10 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - storage, "#{path}.git", "#{new_path}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace @@ -130,10 +136,11 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - forked_from_storage, "#{path}.git", forked_to_storage, - fork_namespace]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', + forked_from_storage, "#{path}.git", forked_to_storage, + fork_namespace]) end # Remove repository from file system @@ -144,9 +151,10 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'rm-project', storage, "#{name}.git"]) end # Add new key to gitlab-shell @@ -155,8 +163,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, self.class.strip_key(key_content)]) + gitlab_shell_fast_execute([gitlab_shell_keys_path, + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -175,8 +183,10 @@ module Gitlab # remove_key("key-342", "sha-rsa ...") # def remove_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'rm-key', key_id, key_content]) + args = [gitlab_shell_keys_path, 'rm-key', key_id] + args << key_content if key_content + + gitlab_shell_fast_execute(args) end # Remove all ssh keys from gitlab shell @@ -185,7 +195,7 @@ module Gitlab # remove_all_keys # def remove_all_keys - Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear']) + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end # Add empty directory for storing repositories @@ -193,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -206,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -215,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -240,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end @@ -267,5 +281,31 @@ module Gitlab def gitlab_shell_keys_path File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + + private + + def gitlab_shell_fast_execute(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + return true if status.zero? + + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + false + end + + def gitlab_shell_fast_execute_raise_error(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + raise Error, output unless status.zero? + true + end + + def gitlab_shell_fast_execute_helper(cmd) + vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS) + + # Don't pass along the entire parent environment to prevent gitlab-shell + # from wasting I/O by searching through GEM_PATH + Bundler.with_original_env { Popen.popen(cmd, nil, vars) } + end end end diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index aa1468bff6b..b5f9d040047 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -77,8 +77,8 @@ module Gitlab line_samples << LineSample.new(duration, events) end - samples << FileSample. - new(file, line_samples, total_duration, total_events) + samples << FileSample + .new(file, line_samples, total_duration, total_events) end samples diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 99e56e923eb..948bf5e6528 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -105,10 +105,10 @@ module Gitlab end def format_sql(query) - query.each_line. - map { |line| line.strip }. - join("\n"). - gsub(PREFIX_NEWLINE) { "\n#{$1} " } + query.each_line + .map { |line| line.strip } + .join("\n") + .gsub(PREFIX_NEWLINE) { "\n#{$1} " } end end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index 25da8474e95..cc3c9a50555 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class BaseCommand QUERY_LIMIT = 5 diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 3e0c30c33b7..a78408b0519 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,11 +1,11 @@ module Gitlab - module ChatCommands + module SlashCommands class Command < BaseCommand COMMANDS = [ - Gitlab::ChatCommands::IssueShow, - Gitlab::ChatCommands::IssueNew, - Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::Deploy ].freeze def execute @@ -15,10 +15,10 @@ module Gitlab if command.allowed?(project, current_user) command.new(project, current_user, params).execute(match) else - Gitlab::ChatCommands::Presenters::Access.new.access_denied + Gitlab::SlashCommands::Presenters::Access.new.access_denied end else - Gitlab::ChatCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + Gitlab::SlashCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 458d90f84e8..93e00ab75a1 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class Deploy < BaseCommand def self.match(text) /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) @@ -21,29 +21,34 @@ module Gitlab from = match[:from] to = match[:to] - actions = find_actions(from, to) + action = find_action(from, to) - if actions.none? - Gitlab::ChatCommands::Presenters::Deploy.new(nil).no_actions - elsif actions.one? - action = play!(from, to, actions.first) - Gitlab::ChatCommands::Presenters::Deploy.new(action).present(from, to) + if action.nil? + Gitlab::SlashCommands::Presenters::Deploy + .new(action).action_not_found else - Gitlab::ChatCommands::Presenters::Deploy.new(actions).too_many_actions + deployment = action.play(current_user) + + Gitlab::SlashCommands::Presenters::Deploy + .new(deployment).present(from, to) end end private - def play!(from, to, action) - action.play(current_user) - end - - def find_actions(from, to) + def find_action(from, to) environment = project.environments.find_by(name: from) - return [] unless environment + return unless environment - environment.actions_for(to).select(&:starts_environment?) + actions = environment.actions_for(to).select do |action| + action.starts_environment? + end + + if actions.many? + actions.find { |action| action.name == to.to_s } + else + actions.first + end end end end diff --git a/lib/gitlab/chat_commands/help.rb b/lib/gitlab/slash_commands/help.rb index 6c0e4d304a4..81f3707e03e 100644 --- a/lib/gitlab/chat_commands/help.rb +++ b/lib/gitlab/slash_commands/help.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class Help < BaseCommand # This class has to be used last, as it always matches. It has to match # because other commands were not triggered and we want to show the help @@ -17,7 +17,7 @@ module Gitlab end def execute(commands, text) - Gitlab::ChatCommands::Presenters::Help.new(commands).present(trigger, text) + Gitlab::SlashCommands::Presenters::Help.new(commands).present(trigger, text) end def trigger diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb index 84de3e44c70..3d96982b820 100644 --- a/lib/gitlab/chat_commands/issue_command.rb +++ b/lib/gitlab/slash_commands/issue_command.rb @@ -1,8 +1,8 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueCommand < BaseCommand def self.available?(project) - project.issues_enabled? && project.default_issues_tracker? + project.issues_enabled? end def collection diff --git a/lib/gitlab/chat_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 016054ecd46..25f965e843d 100644 --- a/lib/gitlab/chat_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueNew < IssueCommand def self.match(text) # we can not match \n with the dot by passing the m modifier as than @@ -35,7 +35,7 @@ module Gitlab end def presenter(issue) - Gitlab::ChatCommands::Presenters::IssueNew.new(issue) + Gitlab::SlashCommands::Presenters::IssueNew.new(issue) end end end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb index 3491b53093e..acba84b54b4 100644 --- a/lib/gitlab/chat_commands/issue_search.rb +++ b/lib/gitlab/slash_commands/issue_search.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueSearch < IssueCommand def self.match(text) /\Aissue\s+search\s+(?<query>.*)/.match(text) diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/slash_commands/issue_show.rb index d6013f4d10c..ffa5184e5cb 100644 --- a/lib/gitlab/chat_commands/issue_show.rb +++ b/lib/gitlab/slash_commands/issue_show.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands class IssueShow < IssueCommand def self.match(text) /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) @@ -13,9 +13,9 @@ module Gitlab issue = find_by_iid(match[:iid]) if issue - Gitlab::ChatCommands::Presenters::IssueShow.new(issue).present + Gitlab::SlashCommands::Presenters::IssueShow.new(issue).present else - Gitlab::ChatCommands::Presenters::Access.new.not_found + Gitlab::SlashCommands::Presenters::Access.new.not_found end end end diff --git a/lib/gitlab/chat_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 92f4fa17f78..1a817eb735b 100644 --- a/lib/gitlab/chat_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Access < Presenters::Base def access_denied diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 05994bee79d..e13808a2720 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -1,8 +1,8 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Base - include Gitlab::Routing.url_helpers + include Gitlab::Routing def initialize(resource = nil) @resource = resource diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb new file mode 100644 index 00000000000..ebae0f57f9b --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -0,0 +1,18 @@ +module Gitlab + module SlashCommands + module Presenters + class Deploy < Presenters::Base + def present(from, to) + message = "Deployment started from #{from} to #{to}. " \ + "[Follow its progress](#{resource_url})." + + in_channel_response(text: message) + end + + def action_not_found + ephemeral_response(text: "Couldn't find a deployment manual action.") + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index cd47b7f4c6a..ea611a4d629 100644 --- a/lib/gitlab/chat_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class Help < Presenters::Base def present(trigger, text) diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 25bc82994ba..341f2aabdd0 100644 --- a/lib/gitlab/chat_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters module IssueBase def color(issuable) diff --git a/lib/gitlab/chat_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 3674ba25641..86490a39cc1 100644 --- a/lib/gitlab/chat_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueNew < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb index 73788cf9662..4e27d668685 100644 --- a/lib/gitlab/chat_commands/presenters/issue_search.rb +++ b/lib/gitlab/slash_commands/presenters/issue_search.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueSearch < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index bd784ad241e..c99316df667 100644 --- a/lib/gitlab/chat_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands module Presenters class IssueShow < Presenters::Base include Presenters::IssueBase diff --git a/lib/gitlab/chat_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 324d7ef43a3..7021b4b01b2 100644 --- a/lib/gitlab/chat_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,5 +1,5 @@ module Gitlab - module ChatCommands + module SlashCommands Result = Struct.new(:type, :message) end end diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb new file mode 100644 index 00000000000..5e89e12b2b1 --- /dev/null +++ b/lib/gitlab/sql/glob.rb @@ -0,0 +1,22 @@ +module Gitlab + module SQL + module Glob + extend self + + # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern + # with SQL expression + def to_like(pattern) + <<~SQL + REPLACE(REPLACE(REPLACE(#{pattern}, + #{q('%')}, #{q('\\%')}), + #{q('_')}, #{q('\\_')}), + #{q('*')}, #{q('%')}) + SQL + end + + def q(string) + ActiveRecord::Base.connection.quote(string) + end + end + end +end diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index 5b1b03820a3..16ec002f139 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -52,10 +52,10 @@ module Gitlab # Applies the CTE to the given relation, returning a new one that will # query from it. def apply_to(relation) - relation.except(:where). - with. - recursive(to_arel). - from(alias_to(relation.model.arel_table)) + relation.except(:where) + .with + .recursive(to_arel) + .from(alias_to(relation.model.arel_table)) end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb new file mode 100644 index 00000000000..7ce2e9d636e --- /dev/null +++ b/lib/gitlab/untrusted_regexp.rb @@ -0,0 +1,49 @@ +module Gitlab + # An untrusted regular expression is any regexp containing patterns sourced + # from user input. + # + # Ruby's built-in regular expression library allows patterns which complete in + # exponential time, permitting denial-of-service attacks. + # + # Not all regular expression features are available in untrusted regexes, and + # there is a strict limit on total execution time. See the RE2 documentation + # at https://github.com/google/re2/wiki/Syntax for more details. + class UntrustedRegexp + delegate :===, to: :regexp + + def initialize(pattern) + @regexp = RE2::Regexp.new(pattern, log_errors: false) + + raise RegexpError.new(regexp.error) unless regexp.ok? + end + + def replace_all(text, rewrite) + RE2.GlobalReplace(text, regexp, rewrite) + end + + def scan(text) + matches = scan_regexp.scan(text).to_a + matches.map!(&:first) if regexp.number_of_capturing_groups.zero? + matches + end + + def replace(text, rewrite) + RE2.Replace(text, regexp, rewrite) + end + + private + + attr_reader :regexp + + # RE2 scan operates differently to Ruby scan when there are no capture + # groups, so work around it + def scan_regexp + @scan_regexp ||= + if regexp.number_of_capturing_groups.zero? + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end + end + end +end diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index 7d0c47c5361..b5f41240529 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,7 +1,7 @@ module Gitlab class UploadsTransfer < ProjectTransfer def root_dir - File.join(CarrierWave.root, GitlabUploader.base_dir) + File.join(CarrierWave.root, FileUploader.base_dir) end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index ccb456bcc94..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Gitlab::Routing.url_helpers + include Gitlab::Routing include GitlabRoutingHelper include ActionView::RecordIdentifier @@ -23,9 +23,9 @@ module Gitlab when WikiPage wiki_page_url when ProjectSnippet - project_snippet_url(object) + project_snippet_url(object.project, object) when Snippet - personal_snippet_url(object) + snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end @@ -52,21 +52,24 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) - project_snippet_url(snippet, anchor: dom_id(object)) + snippet = object.noteable + + if snippet.is_a?(PersonalSnippet) + snippet_url(snippet, anchor: dom_id(object)) + else + project_snippet_url(snippet.project, snippet, anchor: dom_id(object)) + end end end def wiki_page_url - namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + project_wiki_url(object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index bcba2e3e1b6..e0ac21305a5 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -20,13 +20,15 @@ module Gitlab counts: { boards: Board.count, ci_builds: ::Ci::Build.count, - ci_pipelines: ::Ci::Pipeline.count, + ci_internal_pipelines: ::Ci::Pipeline.internal.count, + ci_external_pipelines: ::Ci::Pipeline.external.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, ci_pipeline_schedules: ::Ci::PipelineSchedule.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: Environment.count, + in_review_folder: Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, keys: Key.count, @@ -37,14 +39,14 @@ module Gitlab notes: Note.count, pages_domains: PagesDomain.count, projects: Project.count, - projects_prometheus_active: PrometheusService.active.count, + projects_imported_from_github: Project.where(import_type: 'github').count, protected_branches: ProtectedBranch.count, releases: Release.count, snippets: Snippet.count, todos: Todo.count, uploads: Upload.count, web_hooks: WebHook.count - } + }.merge(services_usage) } end @@ -61,6 +63,18 @@ module Gitlab usage_data end + + def services_usage + types = { + JiraService: :projects_jira_active, + SlackService: :projects_slack_notifications_active, + SlackSlashCommandsService: :projects_slack_slash_active, + PrometheusService: :projects_prometheus_active + } + + results = Service.unscoped.where(type: types.keys, active: true).group(:type).count + results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + end end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 3b922da7ced..d9a5af09f08 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,5 +1,11 @@ module Gitlab class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + attr_reader :user, :project def initialize(user, project: nil) @@ -28,43 +34,47 @@ module Gitlab true end - def can_create_tag?(ref) + request_cache def can_create_tag?(ref) return false unless can_access_git? - if ProtectedTag.protected?(project, ref) - project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + if protected?(ProtectedTag, project, ref) + protected_tag_accessible_to?(ref, action: :create) else user.can?(:push_code, project) end end - def can_delete_branch?(ref) + request_cache def can_delete_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) user.can?(:delete_protected_branch, project) else user.can?(:push_code, project) end end - def can_push_to_branch?(ref) + def can_update_branch?(ref) + can_push_to_branch?(ref) || can_merge_to_branch?(ref) + end + + request_cache def can_push_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) + protected_branch_accessible_to?(ref, action: :push) else user.can?(:push_code, project) end end - def can_merge_to_branch?(ref) + request_cache def can_merge_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) + if protected?(ProtectedBranch, project, ref) + protected_branch_accessible_to?(ref, action: :merge) else user.can?(:push_code, project) end @@ -81,5 +91,23 @@ module Gitlab def can_access_git? user && user.can?(:access_git) end + + def protected_branch_accessible_to?(ref, action:) + ProtectedBranch.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_branches) + end + + def protected_tag_accessible_to?(ref, action:) + ProtectedTag.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_tags) + end + + request_cache def protected?(kind, project, ref) + kind.protected?(project, ref) + end end end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index dbfe0941e4d..841fb681435 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -15,6 +15,11 @@ module Gitlab super(user, action, overriden_subject || subject) end + # delegate all #can? queries to the subject + def declarative_policy_delegate + subject + end + class_methods do def presenter? true diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 85da4c8660b..c60bd91ea6e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -13,18 +13,8 @@ module Gitlab scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } - scope :public_to_user, -> (user) do - if user - if user.admin? - all - elsif !user.external? - public_and_internal_only - else - public_only - end - else - public_only - end + scope :public_to_user, -> (user = nil) do + where(visibility_level: VisibilityLevel.levels_for_user(user)) end end @@ -35,15 +25,27 @@ module Gitlab class << self delegate :values, to: :options + def levels_for_user(user = nil) + return [PUBLIC] unless user + + if user.full_private_access? + [PRIVATE, INTERNAL, PUBLIC] + elsif user.external? + [PUBLIC] + else + [INTERNAL, PUBLIC] + end + end + def string_values string_options.keys end def options { - 'Private' => PRIVATE, - 'Internal' => INTERNAL, - 'Public' => PUBLIC + N_('VisibilityLevel|Private') => PRIVATE, + N_('VisibilityLevel|Internal') => INTERNAL, + N_('VisibilityLevel|Public') => PUBLIC } end @@ -87,12 +89,12 @@ module Gitlab end def level_name(level) - level_name = 'Unknown' + level_name = N_('VisibilityLevel|Unknown') options.each do |name, lvl| level_name = name if lvl == level.to_i end - level_name + s_(level_name) end def level_value(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 7f27317775c..3f25e463412 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -25,22 +25,28 @@ module Gitlab RepoPath: repo_path } - if Gitlab.config.gitaly.enabled - address = Gitlab::GitalyClient.address(project.repository_storage) - params[:Repository] = repository.gitaly_repository.to_h - - feature_enabled = case action.to_s - when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) - when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) - when 'info_refs' - true - else - raise "Unsupported action: #{action}" - end - - params[:GitalyAddress] = address if feature_enabled + server = { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?( + :post_upload_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + if feature_enabled + params[:GitalyAddress] = server[:address] # This field will be deprecated + params[:GitalyServer] = server end params @@ -59,10 +65,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -173,7 +190,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -189,6 +206,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end |