diff options
Diffstat (limited to 'lib/gitlab')
154 files changed, 3278 insertions, 1477 deletions
diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index e4f7cad2b79..45c2b01dd8f 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -1,7 +1,7 @@ module Gitlab module Allowable - def can?(user, action, subject = :global) - Ability.allowed?(user, action, subject) + def can?(*args) + Ability.allowed?(*args) end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 3933c3b04dd..9bed81e7327 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,7 +37,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -48,6 +48,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -130,13 +134,13 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES) Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_scoped_token?(token, ["api"]) + token && token.accessible? && valid_scoped_token?(token, [:api]) end def valid_scoped_token?(token, scopes) diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d95ecd7b291..d3f66877672 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,24 +1,45 @@ module Gitlab module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue - .new(BackgroundMigrationWorker.sidekiq_options['queue']) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - queue.each do |job| - migration_class, migration_args = job.args + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args - next unless migration_class == steal_class + next unless job.queue == self.queue + next unless migration_class == steal_class - perform(migration_class, migration_args) + begin + perform(migration_class, migration_args) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) - job.delete + raise + end + end end end + ## + # Performs a background migration. + # # class_name - The name of the background migration class as defined in the # Gitlab::BackgroundMigration namespace. # diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb new file mode 100644 index 00000000000..91540127ea9 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb @@ -0,0 +1,19 @@ +module Gitlab + module BackgroundMigration + class MigrateBuildStageIdReference + def perform(start_id, stop_id) + sql = <<-SQL.strip_heredoc + UPDATE ci_builds + SET stage_id = + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND ci_builds.stage_id IS NULL + SQL + + ActiveRecord::Base.connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb new file mode 100644 index 00000000000..0881244ed49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -0,0 +1,26 @@ +module Gitlab + module BackgroundMigration + class MigrateSystemUploadsToNewFolder + include Gitlab::Database::MigrationHelpers + attr_reader :old_folder, :new_folder + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + include EachBatch + end + + def perform(old_folder, new_folder) + replace_sql = replace_sql(uploads[:path], old_folder, new_folder) + affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) + + affected_uploads.each_batch do |batch| + batch.update_all("path = #{replace_sql}") + end + end + + def uploads + Arel::Table.new('uploads') + end + end + end +end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb index 53588185622..e898f5d790e 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -16,13 +16,11 @@ module Gitlab end def image_url - coverage_namespace_project_badges_url(@project.namespace, - @project, @ref, - format: :svg) + coverage_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, @ref) end end end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 4a049ef758d..8ad6f3cb986 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -4,7 +4,7 @@ module Gitlab # Abstract class for badge metadata # class Metadata - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing include ActionView::Helpers::AssetTagHelper include ActionView::Helpers::UrlHelper diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/pipeline/metadata.rb index f87a7b7942e..db1e9f8cfb8 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/pipeline/metadata.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that describes build badge metadata + # Class that describes pipeline badge metadata # class Metadata < Badge::Metadata def initialize(badge) @@ -11,16 +11,15 @@ module Gitlab end def title - 'build status' + 'pipeline status' end def image_url - build_namespace_project_badges_url(@project.namespace, - @project, @ref, format: :svg) + pipeline_project_badges_url(@project, @ref, format: :svg) end def link_url - namespace_project_commits_url(@project.namespace, @project, id: @ref) + project_commits_url(@project, id: @ref) end end end diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/pipeline/status.rb index b762d85b6e5..5fee7a93475 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -1,8 +1,8 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Build status badge + # Pipeline status badge # class Status < Badge::Base attr_reader :project, :ref @@ -15,7 +15,7 @@ module Gitlab end def entity - 'build' + 'pipeline' end def status @@ -25,11 +25,11 @@ module Gitlab end def metadata - @metadata ||= Build::Metadata.new(self) + @metadata ||= Pipeline::Metadata.new(self) end def template - @template ||= Build::Template.new(self) + @template ||= Pipeline::Template.new(self) end end end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/pipeline/template.rb index bc0e0cd441d..e09db32262d 100644 --- a/lib/gitlab/badge/build/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -1,12 +1,13 @@ module Gitlab module Badge - module Build + module Pipeline ## - # Class that represents a build badge template. + # Class that represents a pipeline badge template. # # Template object will be passed to badge.svg.erb template. # class Template < Badge::Template + STATUS_RENAME = { 'success' => 'passed' }.freeze STATUS_COLOR = { success: '#4c1', failed: '#e05d44', @@ -27,11 +28,11 @@ module Gitlab end def value_text - @status.to_s + STATUS_RENAME[@status.to_s] || @status.to_s end def key_width - 38 + 62 end def value_width diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb new file mode 100644 index 00000000000..f1a04affd38 --- /dev/null +++ b/lib/gitlab/cache/request_cache.rb @@ -0,0 +1,94 @@ +module Gitlab + module Cache + # This module provides a simple way to cache values in RequestStore, + # and the cache key would be based on the class name, method name, + # optionally customized instance level values, optionally customized + # method level values, and optional method arguments. + # + # A simple example: + # + # class UserAccess + # extend Gitlab::Cache::RequestCache + # + # request_cache_key do + # [user&.id, project&.id] + # end + # + # request_cache def can_push_to_branch?(ref) + # # ... + # end + # end + # + # This way, the result of `can_push_to_branch?` would be cached in + # `RequestStore.store` based on the cache key. If RequestStore is not + # currently active, then it would be stored in a hash saved in an + # instance variable, so the cache logic would be the same. + # Here's another example using customized method level values: + # + # class Commit + # extend Gitlab::Cache::RequestCache + # + # def author + # User.find_by_any_email(author_email.downcase) + # end + # request_cache(:author) { author_email.downcase } + # end + # + # So that we could have different strategies for different methods + # + module RequestCache + def self.extended(klass) + return if klass < self + + extension = Module.new + klass.const_set(:RequestCacheExtension, extension) + klass.prepend(extension) + end + + def request_cache_key(&block) + if block_given? + @request_cache_key = block + else + @request_cache_key + end + end + + def request_cache(method_name, &method_key_block) + const_get(:RequestCacheExtension).module_eval do + cache_key_method_name = "#{method_name}_cache_key" + + define_method(method_name) do |*args| + store = + if RequestStore.active? + RequestStore.store + else + ivar_name = # ! and ? cannot be used as ivar name + "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" + + instance_variable_get(ivar_name) || + instance_variable_set(ivar_name, {}) + end + + key = __send__(cache_key_method_name, args) + + store.fetch(key) { store[key] = super(*args) } + end + + define_method(cache_key_method_name) do |args| + klass = self.class + + instance_key = instance_exec(&klass.request_cache_key) if + klass.request_cache_key + + method_key = instance_exec(&method_key_block) if method_key_block + + [klass.name, method_name, *instance_key, *method_key, *args] + .join(':') + end + + private cache_key_method_name + end + end + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f074df9c7a1..d7e09acbbf3 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,11 +7,14 @@ module Gitlab # class Cache < Node include Configurable + include Attributable - ALLOWED_KEYS = %i[key untracked paths].freeze + ALLOWED_KEYS = %i[key untracked paths policy].freeze + DEFAULT_POLICY = 'pull-push'.freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true end entry :key, Entry::Key, @@ -25,8 +28,15 @@ module Gitlab helpers :key + attributes :policy + def value - super.merge(key: key_value) + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + + result end end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 897dcff8012..6555c589173 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -15,7 +15,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :name, type: String, presence: true - validates :entrypoint, type: String, allow_nil: true + validates :entrypoint, array_of_strings: true, allow_nil: true end def hash? diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 176301bcca1..32f5c6ab142 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment coverage].freeze + after_script variables environment coverage retry].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -23,6 +23,9 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true + validates :retry, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ @@ -76,9 +79,9 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage + :artifacts, :commands, :environment, :coverage, :retry - attributes :script, :tags, :allow_failure, :when, :dependencies + attributes :script, :tags, :allow_failure, :when, :dependencies, :retry def compose!(deps = nil) super do @@ -142,6 +145,7 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, + retry: retry_defined? ? retry_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index b52faf48b58..3e2ebcff31a 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,8 +15,8 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :name, type: String, presence: true - validates :entrypoint, type: String, allow_nil: true - validates :command, type: String, allow_nil: true + validates :entrypoint, array_of_strings: true, allow_nil: true + validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 439ef0ce015..8ad3e57e59d 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,9 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + cancel_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index b173c23fba4..c0c7c7f5b5d 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + project_job_path(subject.project, subject) end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index e80f3263794..c7726543599 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 56303e4cb17..8c8fdc56d75 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,9 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + retry_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 2778d6f3b52..d464738deaf 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,9 +20,7 @@ module Gitlab end def action_path - play_namespace_project_job_path(subject.project.namespace, - subject.project, - subject) + play_project_job_path(subject.project, subject) end def action_method diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 76bfd18bf40..61bb07beb0f 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -8,9 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject) + project_pipeline_path(subject.project, subject) end def has_action? diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index 7852f492e1d..bc99d925347 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -8,10 +8,7 @@ module Gitlab end def details_path - namespace_project_pipeline_path(subject.project.namespace, - subject.project, - subject.pipeline, - anchor: subject.name) + project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) end def has_action? diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c4c0623df6c..8503ecf8700 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -67,14 +67,15 @@ module Gitlab def extract_coverage(regex) return unless valid? - return unless regex + return unless regex.present? - regex = Regexp.new(regex) + regex = Gitlab::UntrustedRegexp.new(regex) match = "" reverse_line do |line| - matches = line.scan(regex) + line.chomp! + matches = regex.scan(line) next unless matches.is_a?(Array) next if matches.empty? diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 75a213ef752..98dfe900044 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,7 +1,7 @@ module Gitlab module Conflict class File - include Gitlab::Routing.url_helpers + include Gitlab::Routing include IconsHelper MissingResolution = Class.new(ResolutionError) @@ -205,9 +205,7 @@ module Gitlab old_path: their_path, new_path: our_path, blob_icon: file_type_icon_class('file', our_mode, our_path), - blob_path: namespace_project_blob_path(merge_request.project.namespace, - merge_request.project, - ::File.join(merge_request.diff_refs.head_sha, our_path)) + blob_path: project_blob_path(merge_request.project, ::File.join(merge_request.diff_refs.head_sha, our_path)) } json_hash.tap do |json_hash| @@ -223,11 +221,10 @@ module Gitlab end def content_path - conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, - merge_request.project, - merge_request, - old_path: their_path, - new_path: our_path) + conflict_for_path_project_merge_request_path(merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) end # Don't try to print merge_request or repository. diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 818b3d9c46b..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available end end @@ -33,12 +33,7 @@ module Gitlab def uncached_application_settings return fake_application_settings unless connect_to_db? - # This loads from the database into the cache, so handle Redis errors - begin - db_settings = ::ApplicationSetting.current - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - end + db_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 9d25ef078e8..f5d08c0b658 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -13,6 +13,10 @@ module Gitlab MergeRequestDiff.arel_table end + def mr_diff_commits_table + MergeRequestDiffCommit.arel_table + end + def mr_closing_issues_table MergeRequestsClosingIssues.arel_table end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 7d342a2d2cb..b260822788d 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -2,40 +2,59 @@ module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher def initialize(*args) - @projections = [mr_diff_table[:st_commits].as('commits'), + @projections = [mr_diff_table[:id], + mr_diff_table[:st_commits], issue_metrics_table[:first_mentioned_in_commit_at]] super(*args) end def events_query - base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + base_query + .join(mr_diff_table) + .on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) super end private + def merge_request_diff_commits + @merge_request_diff_commits ||= + MergeRequestDiffCommit + .where(merge_request_diff_id: event_result.map { |event| event['id'] }) + .group_by(&:merge_request_diff_id) + end + def serialize(event) - st_commit = first_time_reference_commit(event.delete('commits'), event) + commit = first_time_reference_commit(event) - return unless st_commit + return unless commit - serialize_commit(event, st_commit, query) + serialize_commit(event, commit, query) end - def first_time_reference_commit(commits, event) + def first_time_reference_commit(event) + return nil unless event && merge_request_diff_commits + + commits = + if event['st_commits'].present? + YAML.load(event['st_commits']) + else + merge_request_diff_commits[event['id'].to_i] + end + return nil if commits.blank? - YAML.load(commits).find do |commit| + commits.find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i end end - def serialize_commit(event, st_commit, query) - commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + def serialize_commit(event, commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(commit.to_hash), @project) AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index e81d19a7a2e..5c5f507d44d 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -24,11 +24,11 @@ module Gitlab # total_commits_count: Fixnum # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil) + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) commits = Array(commits) # Total commits count - commits_count = commits.size + commits_count ||= commits.size # Get latest 20 commits ASC commits_limited = commits.last(20) @@ -74,6 +74,8 @@ module Gitlab build(project, user, commits.last&.id, commits.first&.id, ref, commits) end + private + def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb new file mode 100644 index 00000000000..226974b698c --- /dev/null +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -0,0 +1,22 @@ +module Gitlab + module DataBuilder + module WikiPage + extend self + + def build(wiki_page, user, action) + wiki = wiki_page.wiki + + { + object_kind: wiki_page.class.name.underscore, + user: user.hook_attrs, + project: wiki.project.hook_attrs, + wiki: wiki.hook_attrs, + object_attributes: wiki_page.hook_attrs.merge( + url: Gitlab::UrlBuilder.build(wiki_page), + action: action + ) + } + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 0d5a7cf0694..d7dab584a44 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -93,7 +93,7 @@ module Gitlab row.values_at(*keys).map { |value| connection.quote(value) } end - connection.execute <<-EOF.strip_heredoc + connection.execute <<-EOF INSERT INTO #{table} (#{columns.join(', ')}) VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 60cce9c6d9e..69ca9aa596b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -140,6 +140,8 @@ module Gitlab return add_foreign_key(source, target, column: column, on_delete: on_delete) + else + on_delete = 'SET NULL' if on_delete == :nullify end disable_statement_timeout @@ -155,7 +157,7 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete}" : ''} + #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} NOT VALID; EOF @@ -222,6 +224,12 @@ module Gitlab # # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) + if transaction_open? + raise 'update_column_in_batches can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + table = Arel::Table.new(table) count_arel = table.project(Arel.star.count.as('count')) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb index 89530082cd2..f333ff22300 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -29,6 +29,11 @@ module Gitlab paths = Array(paths) RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) end + + def revert_renames + RenameProjects.new([], self).revert_renames + RenameNamespaces.new([], self).revert_renames + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index d8163d7da11..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -6,7 +6,10 @@ module Gitlab attr_reader :paths, :migration delegate :update_column_in_batches, + :execute, :replace_sql, + :quote_string, + :say, to: :migration def initialize(paths, migration) @@ -26,24 +29,45 @@ module Gitlab new_path = rename_path(namespace_path, old_path) new_full_path = join_routable_path(namespace_path, new_path) + perform_rename(routable, old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def perform_rename(routable, old_full_path, new_full_path) # skips callbacks & validations + new_path = new_full_path.split('/').last routable.class.where(id: routable) .update_all(path: new_path) rename_routes(old_full_path, new_full_path) - - [old_full_path, new_full_path] end def rename_routes(old_full_path, new_full_path) + routes = Route.arel_table + + quoted_old_full_path = quote_string(old_full_path) + quoted_old_wildcard_path = quote_string("#{old_full_path}/%") + + filter = if Database.mysql? + "lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "OR routes.path LIKE '#{quoted_old_wildcard_path}'" + else + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" + end + replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path) - update_column_in_batches(:routes, :path, replace_statement) do |table, query| - path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) - query.where(path_or_children) - end + update = Arel::UpdateManager.new(ActiveRecord::Base) + .table(routes) + .set([[routes[:path], replace_statement]]) + .where(Arel::Nodes::SqlLiteral.new(filter)) + + execute(update.to_sql) end def rename_path(namespace_path, path_was) @@ -86,32 +110,74 @@ module Gitlab def move_folders(directory, old_relative_path, new_relative_path) old_path = File.join(directory, old_relative_path) - return unless File.directory?(old_path) + unless File.directory?(old_path) + say "#{old_path} doesn't exist, skipping" + return + end new_path = File.join(directory, new_relative_path) FileUtils.mv(old_path, new_path) end def remove_cached_html_for_projects(project_ids) - update_column_in_batches(:projects, :description_html, nil) do |table, query| - query.where(table[:id].in(project_ids)) - end - - update_column_in_batches(:issues, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) + project_ids.each do |project_id| + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].eq(project_id)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].eq(project_id)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].eq(project_id)) + end end + end - update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| - query.where(table[:target_project_id].in(project_ids)) + def track_rename(type, old_path, new_path) + key = redis_key_for_type(type) + Gitlab::Redis::SharedState.with do |redis| + redis.lpush(key, [old_path, new_path].to_json) + redis.expire(key, 2.weeks.to_i) end + say "tracked rename: #{key}: #{old_path} -> #{new_path}" + end - update_column_in_batches(:notes, :note_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) + def reverts_for_type(type) + key = redis_key_for_type(type) + + Gitlab::Redis::SharedState.with do |redis| + failed_reverts = [] + + while rename_info = redis.lpop(key) + path_before_rename, path_after_rename = JSON.parse(rename_info) + say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" + begin + yield(path_before_rename, path_after_rename) + rescue StandardError => e + failed_reverts << rename_info + say "Renaming #{type} from #{path_after_rename} back to "\ + "#{path_before_rename} failed. Review the error and try "\ + "again by running the `down` action. \n"\ + "#{e.message}: \n #{e.backtrace.join("\n")}" + end + end + + failed_reverts.each { |rename_info| redis.lpush(key, rename_info) } end + end - update_column_in_batches(:milestones, :description_html, nil) do |table, query| - query.where(table[:project_id].in(project_ids)) - end + def redis_key_for_type(type) + "rename:#{migration.name}:#{type}" end def file_storage? diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index da7e2cb2e85..05b86f32ce2 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -26,6 +26,12 @@ module Gitlab def rename_namespace(namespace) old_full_path, new_full_path = rename_path_for_routable(namespace) + track_rename('namespace', old_full_path, new_full_path) + + rename_namespace_dependencies(namespace, old_full_path, new_full_path) + end + + def rename_namespace_dependencies(namespace, old_full_path, new_full_path) move_repositories(namespace, old_full_path, new_full_path) move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) @@ -33,6 +39,23 @@ module Gitlab remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) end + def revert_renames + reverts_for_type('namespace') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + namespace = MigrationClasses::Namespace.joins(:route) + .where(matches_path).first&.becomes(MigrationClasses::Namespace) + + if namespace + perform_rename(namespace, current_path, path_before_rename) + + rename_namespace_dependencies(namespace, current_path, path_before_rename) + else + say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\ + "namespace was renamed, or no longer exists at the expected path" + end + end + end + def rename_user(old_username, new_username) MigrationClasses::User.where(username: old_username) .update_all(username: new_username) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 448717eb744..75a75f61953 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -16,12 +16,37 @@ module Gitlab def rename_project(project) old_full_path, new_full_path = rename_path_for_routable(project) + track_rename('project', old_full_path, new_full_path) + + move_project_folders(project, old_full_path, new_full_path) + end + + def move_project_folders(project, old_full_path, new_full_path) move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") move_uploads(old_full_path, new_full_path) move_pages(old_full_path, new_full_path) end + def revert_renames + reverts_for_type('project') do |path_before_rename, current_path| + matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) + project = MigrationClasses::Project.joins(:route) + .where(matches_path).first + + if project + perform_rename(project, current_path, path_before_rename) + + move_project_folders(project, current_path, path_before_rename) + else + say "Couldn't rename project from #{current_path} back to "\ + "#{path_before_rename}, project was renamed or no longer "\ + "exists at the expected path." + + end + end + end + def move_repository(project, old_path, new_path) unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb new file mode 100644 index 00000000000..d9400e04b83 --- /dev/null +++ b/lib/gitlab/database/sha_attribute.rb @@ -0,0 +1,34 @@ +module Gitlab + module Database + BINARY_TYPE = if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + ActiveRecord::Type::Binary + end + + # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). + # + # Using ShaAttribute allows you to store SHA1 values as binary while still + # using them as if they were stored as string values. This gives you the + # ease of use of string values, but without the storage overhead. + class ShaAttribute < BINARY_TYPE + PACK_FORMAT = 'H*'.freeze + + # Casts binary data to a SHA1 in hexadecimal. + def type_cast_from_database(value) + value = super + + value ? value.unpack(PACK_FORMAT)[0] : nil + end + + # Casts a SHA1 in hexadecimal to the proper binary format. + def type_cast_for_database(value) + arg = value ? [value].pack(PACK_FORMAT) : nil + + super(arg) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 7bbd154eb03..d2360583741 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -52,7 +52,7 @@ module Gitlab # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| - marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| url = yield(text) diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index 2e197e5cd94..9c9620bc36a 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -6,7 +6,7 @@ module Gitlab private def link_dependencies - link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name| + link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=~!;\[]+)/) do |name| "https://pypi.python.org/pypi/#{name}" end diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb deleted file mode 100644 index a78fde9d782..00000000000 --- a/lib/gitlab/devise_failure.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - class DeviseFailure < Devise::FailureApp - protected - - # Override `Devise::FailureApp#request_format` to handle a special case - # - # This tells Devise to handle an unauthenticated `.zip` request as an HTML - # request (i.e., redirect to sign in). - # - # Otherwise, Devise would respond with a 401 Unauthorized with - # `Content-Type: application/zip` and a response body in plaintext, and the - # browser would freak out. - # - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/12944 - def request_format - if request.format == :zip - Mime::Type.lookup_by_extension(:html).ref - else - super - end - end - end -end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 6d326ee213a..85e6db0a689 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -76,9 +76,13 @@ module Gitlab step( "Generating the patch against origin/master in #{patch_path}", - %W[git diff --binary origin/master > #{patch_path}] + %w[git diff --binary origin/master...HEAD] ) do |output, status| - throw(:halt_check, :ko) unless status.zero? && File.exist?(patch_path) + throw(:halt_check, :ko) unless status.zero? + + File.write(patch_path, output) + + throw(:halt_check, :ko) unless File.exist?(patch_path) end end @@ -130,7 +134,15 @@ module Gitlab step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", - %W[git apply --check --3way #{patch_path}] + # Don't use --check here because it can result in a 0-exit status even + # though the patch doesn't apply cleanly, e.g.: + # > git apply --check --3way foo.patch + # error: patch failed: lib/gitlab/ee_compat_check.rb:74 + # Falling back to three-way merge... + # Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts. + # > echo $? + # 0 + %W[git apply --3way #{patch_path}] ) do |output, status| puts output unless status.zero? @@ -145,6 +157,7 @@ module Gitlab status = 0 if failed_files.empty? end + command(%w[git reset --hard]) status end end @@ -224,6 +237,10 @@ module Gitlab branch_name.parameterize << '.patch' end + def patch_url + "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/#{ENV['CI_JOB_ID']}/artifacts/raw/ee_compat_check/patches/#{ce_patch_name}" + end + def step(desc, cmd = nil) puts "\n=> #{desc}\n" @@ -290,14 +307,11 @@ module Gitlab 2. Apply your branch's patch to EE - # In the CE repo - $ git fetch origin master - $ git diff --binary origin/master > #{ce_branch}.patch - # In the EE repo $ git fetch origin master $ git checkout -b #{ee_branch_prefix} origin/master - $ git apply --3way path/to/#{ce_branch}.patch + $ wget #{patch_url} + $ git apply --3way #{ce_patch_name} At this point you might have conflicts such as: @@ -311,7 +325,7 @@ module Gitlab If the patch couldn't be applied cleanly, use the following command: # In the EE repo - $ git apply --reject path/to/#{ce_branch}.patch + $ git apply --reject #{ce_patch_name} This option makes git apply the parts of the patch that are applicable, and leave the rejected hunks in corresponding `.rej` files. @@ -324,7 +338,7 @@ module Gitlab # In the EE repo $ git push origin #{ee_branch_prefix} - ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + ⚠️ Also, don't forget to create a new merge request on gitlab-ee and cross-link it with the CE merge request. Once this is done, you can retry this failed build, and it should pass. diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index ea035e33eff..dd1d9dcd555 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -4,7 +4,7 @@ module Gitlab class RepositoryPush attr_reader :author_id, :ref, :action - include Gitlab::Routing.url_helpers + include Gitlab::Routing include DiffHelper delegate :namespace, :name_with_namespace, to: :project, prefix: :project @@ -96,20 +96,13 @@ module Gitlab def target_url if @action == :push && commits if commits.length > 1 - namespace_project_compare_url(project_namespace, - project, - from: compare.start_commit, - to: compare.head_commit) + project_compare_url(project, from: compare.start_commit, to: compare.head_commit) else - namespace_project_commit_url(project_namespace, - project, - commits.first) + project_commit_url(project, commits.first) end else unless @action == :delete - namespace_project_tree_url(project_namespace, - project, - ref_name) + project_tree_url(project, ref_name) end end end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 62ddd45785d..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -10,25 +10,33 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease - LUA_CANCEL_SCRIPT = <<-EOS.freeze + LUA_CANCEL_SCRIPT = <<~EOS.freeze local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then redis.call("del", key) end EOS + LUA_RENEW_SCRIPT = <<~EOS.freeze + local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2] + if redis.call("get", key) == uuid then + redis.call("expire", key, ttl) + return uuid + end + EOS + def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -37,15 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + end + end + + # Try to renew an existing lease. Return lease UUID on success, + # false if the lease is taken by a different UUID or inexistent. + def renew + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) + result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 936606152e9..b6449f27034 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -7,8 +7,10 @@ module Gitlab CommandError = Class.new(StandardError) class << self + include Gitlab::EncodingHelper + def ref_name(ref) - ref.sub(/\Arefs\/(tags|heads)\//, '') + encode! ref.sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..0deaab01b5b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC for #load_blame. + module Gitlab module Git class Blame @@ -24,6 +26,7 @@ module Gitlab private + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) # Read in binary mode to ensure ASCII-8BIT diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 33a7624e303..db6cfc9671f 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -14,6 +16,47 @@ module Gitlab class << self def find(repository, sha, path) + Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| + if is_enabled + find_by_gitaly(repository, sha, path) + else + find_by_rugged(repository, sha, path) + end + end + end + + def find_by_gitaly(repository, sha, path) + path = path.sub(/\A\/*/, '') + path = '/' if path.empty? + name = File.basename(path) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + return unless entry + + case entry.type + when :COMMIT + new( + id: entry.oid, + name: name, + size: 0, + data: '', + path: path, + commit_id: sha + ) + when :BLOB + new( + id: entry.oid, + name: name, + size: entry.size, + data: entry.data.dup, + mode: entry.mode.to_s(8), + path: path, + commit_id: sha, + binary: binary?(entry.data) + ) + end + end + + def find_by_rugged(repository, sha, path) commit = repository.lookup(sha) root_tree = commit.tree @@ -42,16 +85,32 @@ module Gitlab end def raw(repository, sha) - blob = repository.lookup(sha) + Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + else + blob = repository.lookup(sha) - new( - id: blob.oid, - size: blob.size, - data: blob.content(MAX_DATA_DISPLAY_SIZE), - binary: blob.binary? - ) + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + end end + def binary?(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + detect && detect[:type] == :binary + end + + private + # Recursive search of blob id by path # # Ex. @@ -120,8 +179,17 @@ module Gitlab return if @data == '' # don't mess with submodule blobs return @data if @loaded_all_data + Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + @data = begin + if is_enabled + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + else + repository.lookup(id).content + end + end + end + @loaded_all_data = true - @data = repository.lookup(id).content @loaded_size = @data.bytesize @binary = nil end @@ -130,6 +198,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def truncated? size && (size > loaded_size) end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d5d149f1423..ca7e3a7c4be 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -97,16 +98,89 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.commits_between(base, head).map do |commit| - decorate(commit) + Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.between(base, head) + else + repo.commits_between(base, head).map { |c| decorate(c) } + end end rescue Rugged::ReferenceError [] end - # Delegate Repository#find_commits + # Returns commits collection + # + # Ex. + # Commit.find_all( + # repo, + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - repo.find_commits(options) + Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| + if is_enabled + find_all_by_gitaly(repo, options) + else + find_all_by_rugged(repo, options) + end + end + end + + def find_all_by_rugged(repo, options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + rugged = repo.rugged + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + walker.sorting(rugged_sort_type(actual_options[:order])) + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + commits.push(decorate(commit)) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + def find_all_by_gitaly(repo, options = {}) + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(commit, ref = nil) @@ -131,15 +205,32 @@ module Gitlab diff.find_similar!(break_rewrites: break_rewrites) diff end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE | Rugged::SORT_TOPO + } + + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) + end end def initialize(raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit - if raw_commit.is_a?(Hash) + case raw_commit + when Hash init_from_hash(raw_commit) - elsif raw_commit.is_a?(Rugged::Commit) + when Rugged::Commit init_from_rugged(raw_commit) + when Gitlab::GitalyClient::Commit + init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end @@ -175,8 +266,10 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. - def to_diff(options = {}) - diff_from_parent(options).patch + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 + def to_diff + diff_from_parent.patch end # Returns a diff object for the changes from this commit's first parent. @@ -216,7 +309,23 @@ module Gitlab end def parents - raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + case raw_commit + when Rugged::Commit + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + when Gitlab::GitalyClient::Commit + parent_ids.map { |oid| self.class.find(raw_commit.repository, oid) }.compact + else + raise NotImplementedError, "commit source doesn't support #parents" + end + end + + # Get the gpg signature of this commit. + # + # Ex. + # commit.signature(repo) + # + def signature(repo) + Rugged::Commit.extract_signature(repo.rugged, sha) end def stats @@ -227,7 +336,7 @@ module Gitlab begin raw_commit.to_mbox(options) rescue Rugged::InvalidError => ex - if ex.message =~ /Commit \w+ is a merge commit/ + if ex.message =~ /commit \w+ is a merge commit/i 'Patch format is not currently supported for merge commits.' end end @@ -299,6 +408,22 @@ module Gitlab @parent_ids = commit.parents.map(&:oid) end + def init_from_gitaly(commit) + @raw_commit = commit + @id = commit.id + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + @message = (commit.body.presence || commit.subject).dup + @authored_date = Time.at(commit.author.date.seconds) + @author_name = commit.author.name.dup + @author_email = commit.author.email.dup + @committed_date = Time.at(commit.committer.date.seconds) + @committer_name = commit.committer.name.dup + @committer_email = commit.committer.email.dup + @parent_ids = commit.parent_ids + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 78e440395a5..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index f825568f194..9e00abefd02 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -81,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options @@ -318,7 +226,7 @@ module Gitlab end def init_from_gitaly(diff) - @diff = diff.patch if diff.respond_to?(:patch) + @diff = encode!(diff.patch) if diff.respond_to?(:patch) @new_path = encode!(diff.to_path.dup) @old_path = encode!(diff.from_path.dup) @a_mode = diff.old_mode.to_s(8) @@ -326,6 +234,8 @@ module Gitlab @new_file = diff.from_id == BLANK_SHA @renamed_file = diff.from_path != diff.to_path @deleted_file = diff.to_id == BLANK_SHA + + collapse! if diff.respond_to?(:collapsed) && diff.collapsed end def prune_diff_if_eligible diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 555894907cc..87ed9c3ea26 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection @@ -5,16 +7,28 @@ module Gitlab DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + attr_reader :limits + + delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + + def self.collection_limits(options = {}) + limits = {} + limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file + limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + + OpenStruct.new(limits) + end + def initialize(iterator, options = {}) @iterator = iterator - @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file - @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min - @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @limits = self.class.collection_limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) + @from_gitaly = options.fetch(:from_gitaly, false) @line_count = 0 @byte_count = 0 @@ -24,9 +38,23 @@ module Gitlab end def each(&block) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) + @array.each(&block) + + return if @overflow + return if @iterator.nil? + + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| + if is_enabled && @from_gitaly + each_gitaly_patch(&block) + else + each_rugged_patch(&block) + end end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end def empty? @@ -72,23 +100,32 @@ module Gitlab end def over_safe_limits?(files) - files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes end - def each_patch - i = 0 - @array.each do |diff| - yield diff + def each_gitaly_patch + i = @array.length + + @iterator.each do |raw| + diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded) + + if raw.overflow_marker + @overflow = true + break + end + + yield @array[i] = diff i += 1 end + end - return if @overflow - return if @iterator.nil? + def each_rugged_patch + i = @array.length @iterator.each do |raw| @empty = false - if @enforce_limits && i >= @max_files + if @enforce_limits && i >= max_files @overflow = true break end @@ -104,7 +141,7 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -114,11 +151,6 @@ module Gitlab yield @array[i] = diff i += 1 end - - @populated = true - - # Allow iterator to be garbage-collected. It cannot be reused anyway. - @iterator = nil end end end diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index f4e3b5e5129..4a43b9b444d 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class GitmodulesParser diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index bd90d24a2ec..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,12 +1,17 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook GL_PROTOCOL = 'web'.freeze attr_reader :name, :repo_path, :path - def initialize(name, repo_path) + def initialize(name, project) @name = name - @repo_path = repo_path + @project = project + @repo_path = project.repository.path @path = File.join(repo_path.strip, 'hooks', name) end @@ -38,7 +43,8 @@ module Gitlab vars = { 'GL_ID' => gl_id, 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL + 'GL_PROTOCOL' => GL_PROTOCOL, + 'GL_REPOSITORY' => Gitlab::GlRepository.gl_repository(@project, false) } options = { diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 1add037fa5f..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index @@ -110,10 +114,6 @@ module Gitlab if segment == '..' raise IndexError, 'Path cannot include directory traversal' end - - unless segment =~ Gitlab::Regex.file_name_regex - raise IndexError, "Path #{Gitlab::Regex.file_name_regex_message}" - end end pathname.to_s diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c1f942f931a..a3bc79109f8 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,6 +45,8 @@ module Gitlab :bare?, to: :rugged + delegate :exists?, to: :gitaly_repository_client + # Default branch in the repository def root_ref @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| @@ -80,16 +82,14 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch + def branches + gitaly_migrate(:branches) do |is_enabled| + if is_enabled + gitaly_ref_client.branches + else + branches_filter end - end.compact - - sort_branches(branches, sort_by) + end end def reload_rugged @@ -107,17 +107,18 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) gitaly_migrate(:local_branches) do |is_enabled| if is_enabled - gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch| - Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch) - end + gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -164,20 +165,16 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end + gitaly_migrate(:tags) do |is_enabled| + if is_enabled + tags_from_gitaly + else + tags_from_rugged end - - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) - end.sort_by(&:name) + end end # Returns true if the given tag exists @@ -206,21 +203,10 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end - def repo_exists? - !!rugged - end - # Discovers the default branch based on the repository's available branches # # - If no branches are present, returns nil @@ -299,28 +285,6 @@ module Gitlab (size.to_f / 1024).round(2) end - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) - end - - greps - end - # Use the Rugged Walker API to build an array of commits. # # Usage. @@ -333,85 +297,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -456,7 +345,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -494,70 +383,6 @@ module Gitlab end end - # Returns commits collection - # - # Ex. - # repo.find_commits( - # ref: 'master', - # max_count: 10, - # skip: 5, - # order: :date - # ) - # - # +options+ is a Hash of optional arguments to git - # :ref is the ref from which to begin (SHA1 or name) - # :contains is the commit contained by the refs from which to begin (SHA1 or name) - # :max_count is the maximum number of commits to fetch - # :skip is the number of commits to skip - # :order is the commits order and allowed value is :none (default), :date, - # :topo, or any combination of them (in an array). Commit ordering types - # are documented here: - # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) - # - def find_commits(options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :contains, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) - end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - elsif actual_options[:contains] - branches_contains(actual_options[:contains]).each do |branch| - walker.push(branch.target_id) - end - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end - end - - sort_type = rugged_sort_type(actual_options[:order]) - walker.sorting(sort_type) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - gitlab_commit = Gitlab::Git::Commit.decorate(commit) - commits.push(gitlab_commit) - end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - # Returns branch names collection that contains the special commit(SHA1 # or name) # @@ -613,57 +438,43 @@ module Gitlab rugged.rev_parse(oid_or_ref_name) end - # Return hash with submodules info for this repository + # Returns url for submodule # # Ex. - # { - # "current_path/rack" => { - # "name" => "original_path/rack", - # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", - # "url" => "git://github.com/chneukirchen/rack.git" - # }, - # "encoding" => { - # "id" => .... - # } - # } + # @repository.submodule_url_for('master', 'rack') + # # => git@localhost:rack.git # - def submodules(ref) - commit = rev_parse_target(ref) - return {} unless commit - - begin - content = blob_content(commit, ".gitmodules") - rescue InvalidBlobName - return {} + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 + def submodule_url_for(ref, path) + Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| + if is_enabled + gitaly_submodule_url_for(ref, path) + else + if submodules(ref).any? + submodule = submodules(ref)[path] + submodule['url'] if submodule + end + end end - - parser = GitmodulesParser.new(content) - fill_submodule_ids(commit, parser.parse) end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + gitaly_migrate(:commit_count) do |is_enabled| + if is_enabled + gitaly_commit_client.commit_count(ref) + else + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + end end - # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or - # tag name or a commit SHA. Valid +reset_type+ values are: - # - # [:soft] - # the head will be moved to the commit. - # [:mixed] - # will trigger a +:soft+ reset, plus the index will be replaced - # with the content of the commit tree. - # [:hard] - # will trigger a +:mixed+ reset and the working directory will be - # replaced with the content of the index. (Untracked and ignored files - # will be left alone) - delegate :reset, to: :rugged - # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: # @@ -688,154 +499,6 @@ module Gitlab # TODO: implement this method end - # Check out the specified ref. Valid options are: - # - # :b - Create a new branch at +start_point+ and set HEAD to the new - # branch. - # - # * These options are passed to the Rugged::Repository#checkout method: - # - # :progress :: - # A callback that will be executed for checkout progress notifications. - # Up to 3 parameters are passed on each execution: - # - # - The path to the last updated file (or +nil+ on the very first - # invocation). - # - The number of completed checkout steps. - # - The number of total checkout steps to be performed. - # - # :notify :: - # A callback that will be executed for each checkout notification - # types specified with +:notify_flags+. Up to 5 parameters are passed - # on each execution: - # - # - An array containing the +:notify_flags+ that caused the callback - # execution. - # - The path of the current file. - # - A hash describing the baseline blob (or +nil+ if it does not - # exist). - # - A hash describing the target blob (or +nil+ if it does not exist). - # - A hash describing the workdir blob (or +nil+ if it does not - # exist). - # - # :strategy :: - # A single symbol or an array of symbols representing the strategies - # to use when performing the checkout. Possible values are: - # - # :none :: - # Perform a dry run (default). - # - # :safe :: - # Allow safe updates that cannot overwrite uncommitted data. - # - # :safe_create :: - # Allow safe updates plus creation of missing files. - # - # :force :: - # Allow all updates to force working directory to look like index. - # - # :allow_conflicts :: - # Allow checkout to make safe updates even if conflicts are found. - # - # :remove_untracked :: - # Remove untracked files not in index (that are not ignored). - # - # :remove_ignored :: - # Remove ignored files not in index. - # - # :update_only :: - # Only update existing files, don't create new ones. - # - # :dont_update_index :: - # Normally checkout updates index entries as it goes; this stops - # that. - # - # :no_refresh :: - # Don't refresh index/config/etc before doing checkout. - # - # :disable_pathspec_match :: - # Treat pathspec as simple list of exact match file paths. - # - # :skip_locked_directories :: - # Ignore directories in use, they will be left empty. - # - # :skip_unmerged :: - # Allow checkout to skip unmerged files (NOT IMPLEMENTED). - # - # :use_ours :: - # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). - # - # :use_theirs :: - # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). - # - # :update_submodules :: - # Recursively checkout submodules with same options (NOT - # IMPLEMENTED). - # - # :update_submodules_if_changed :: - # Recursively checkout submodules if HEAD moved in super repo (NOT - # IMPLEMENTED). - # - # :disable_filters :: - # If +true+, filters like CRLF line conversion will be disabled. - # - # :dir_mode :: - # Mode for newly created directories. Default: +0755+. - # - # :file_mode :: - # Mode for newly created files. Default: +0755+ or +0644+. - # - # :file_open_flags :: - # Mode for opening files. Default: - # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. - # - # :notify_flags :: - # A single symbol or an array of symbols representing the cases in - # which the +:notify+ callback should be invoked. Possible values are: - # - # :none :: - # Do not invoke the +:notify+ callback (default). - # - # :conflict :: - # Invoke the callback for conflicting paths. - # - # :dirty :: - # Invoke the callback for "dirty" files, i.e. those that do not need - # an update but no longer match the baseline. - # - # :updated :: - # Invoke the callback for any file that was changed. - # - # :untracked :: - # Invoke the callback for untracked files. - # - # :ignored :: - # Invoke the callback for ignored files. - # - # :all :: - # Invoke the callback for all these cases. - # - # :paths :: - # A glob string or an array of glob strings specifying which paths - # should be taken into account for the checkout operation. +nil+ will - # match all files. Default: +nil+. - # - # :baseline :: - # A Rugged::Tree that represents the current, expected contents of the - # workdir. Default: +HEAD+. - # - # :target_directory :: - # A path to an alternative workdir directory in which the checkout - # should be performed. - def checkout(ref, options = {}, start_point = "HEAD") - if options[:b] - rugged.branches.create(ref, start_point) - options.delete(:b) - end - default_options = { strategy: [:recreate_missing, :safe] } - rugged.checkout(ref, default_options.merge(options)) - end - # Delete the specified branch from the repository def delete_branch(branch_name) rugged.branches.delete(branch_name) @@ -848,7 +511,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -907,6 +571,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -933,6 +598,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -974,8 +640,146 @@ module Gitlab Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) + end + + def gitaly_repository_client + @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) + end + private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + + # We are trying to deprecate this method because it does a lot of work + # but it seems to be used only to look up submodule URL's. + # https://gitlab.com/gitlab-org/gitaly/issues/329 + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parser = GitmodulesParser.new(content) + fill_submodule_ids(commit, parser.parse) + end + + def gitaly_submodule_url_for(ref, path) + # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. + commit_object = gitaly_commit_client.tree_entry(ref, path, 1) + + return unless commit_object && commit_object.type == :COMMIT + + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + found_module = GitmodulesParser.new(gitmodules.data).parse[path] + + found_module && found_module['url'] + end + def alternate_object_directories Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).compact end @@ -1118,73 +922,6 @@ module Gitlab index end - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1213,12 +950,25 @@ module Gitlab end end - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) + def tags_from_rugged + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + end.sort_by(&:name) end - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + def tags_from_gitaly + gitaly_ref_client.tags end def gitaly_migrate(method, &block) @@ -1228,20 +978,6 @@ module Gitlab rescue GRPC::BadStatus => e raise CommandError.new(e) end - - # Returns the `Rugged` sorting type constant for one or more given - # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array - # containing more than one of them. `:date` uses a combination of date and - # topological sorting to closer mimic git's native ordering. - def rugged_sort_type(sort_type) - @rugged_sort_types ||= { - none: Rugged::SORT_NONE, - topo: Rugged::SORT_TOPO, - date: Rugged::SORT_DATE | Rugged::SORT_TOPO - } - - @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) - end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b9afa05c819..8e959c57c7c 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,36 +12,23 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) + Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| + if is_enabled + client = Gitlab::GitalyClient::CommitService.new(repository) + client.tree_entries(repository, sha, path) + else + tree_entries_from_rugged(repository, sha, path) + end end end + private + # Recursive search of tree id for path # # Ex. @@ -68,6 +57,34 @@ module Gitlab entry[:oid] end end + + def tree_entries_from_rugged(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode].to_s(8), + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha + ) + end + end end def initialize(options) @@ -80,6 +97,10 @@ module Gitlab encode! @name end + def path + encode! @path + end + def dir? type == :tree end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f605c06dfc3..c90ef282fdd 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -57,7 +57,7 @@ module Gitlab metadata = yield(metadata) if block_given? stub(service, storage).send(rpc, request, metadata) end - + def self.request_metadata(storage) encoded_token = Base64.strict_encode64(token(storage).to_s) { metadata: { 'authorization' => "Bearer #{encoded_token}" } } @@ -70,12 +70,8 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - def self.enabled? - Gitlab.config.gitaly.enabled - end - def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) - return false if !enabled? || status == MigrationStatus::DISABLED + return false if status == MigrationStatus::DISABLED feature = Feature.get("gitaly_#{feature}") @@ -90,8 +86,8 @@ module Gitlab feature.enabled? end - def self.migrate(feature) - is_enabled = feature_enabled?(feature) + def self.migrate(feature, status: MigrationStatus::OPT_IN) + is_enabled = feature_enabled?(feature, status: status) metric_name = feature.to_s metric_name += "_gitaly" if is_enabled diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb new file mode 100644 index 00000000000..7ea8e8d0857 --- /dev/null +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -0,0 +1,30 @@ +module Gitlab + module GitalyClient + class BlobService + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + end + + def get_blob(oid:, limit:) + request = Gitaly::GetBlobRequest.new( + repository: @gitaly_repo, + oid: oid, + limit: limit + ) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + + blob = response.first + return unless blob.oid.present? + + data = response.reduce(blob.data.dup) { |memo, msg| memo << msg.data.dup } + + Gitlab::Git::Blob.new( + id: blob.oid, + size: blob.size, + data: data, + binary: Gitlab::Git::Blob.binary?(data) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 73c1848c95f..61fe462d762 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -1,52 +1,13 @@ module Gitlab module GitalyClient class Commit - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + attr_reader :repository, :gitaly_commit - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @repository = repository - end - - def is_ancestor(ancestor_id, child_id) - request = Gitaly::CommitIsAncestorRequest.new( - repository: @gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) - - GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value - end - - def diff_from_parent(commit, options = {}) - request_params = commit_diff_request_params(commit, options) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request) - Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) - end - - def commit_deltas(commit) - request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request) - response.flat_map do |msg| - msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } - end - end + delegate :id, :subject, :body, :author, :committer, :parent_ids, to: :gitaly_commit - private - - def commit_diff_request_params(commit, options = {}) - parent_id = commit.parents[0]&.id || EMPTY_TREE_ID - - { - repository: @gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - paths: options.fetch(:paths, []) - } + def initialize(repository, gitaly_commit) + @repository = repository + @gitaly_commit = gitaly_commit end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb new file mode 100644 index 00000000000..c6e52b530b3 --- /dev/null +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -0,0 +1,144 @@ +module Gitlab + module GitalyClient + class CommitService + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + request_params[:enforce_limits] = options.fetch(:limits, true) + request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + + request = Gitaly::CommitDiffRequest.new(request_params) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options.merge(from_gitaly: true)) + end + + def commit_deltas(commit) + request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } + end + end + + def tree_entry(ref, path, limit = nil) + request = Gitaly::TreeEntryRequest.new( + repository: @gitaly_repo, + revision: ref, + path: path.dup.force_encoding(Encoding::ASCII_8BIT), + limit: limit.to_i + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) + entry = response.first + return unless entry.oid.present? + + if entry.type == :BLOB + rest_of_data = response.reduce("") { |memo, msg| memo << msg.data } + entry.data += rest_of_data + end + + entry + end + + def tree_entries(repository, revision, path) + request = Gitaly::GetTreeEntriesRequest.new( + repository: @gitaly_repo, + revision: revision, + path: path.presence || '.' + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request) + + response.flat_map do |message| + message.entries.map do |gitaly_tree_entry| + entry_path = gitaly_tree_entry.path.dup + Gitlab::Git::Tree.new( + id: gitaly_tree_entry.oid, + root_id: gitaly_tree_entry.root_oid, + type: gitaly_tree_entry.type.downcase, + mode: gitaly_tree_entry.mode.to_s(8), + name: File.basename(entry_path), + path: entry_path, + commit_id: gitaly_tree_entry.commit_oid + ) + end + end + end + + def commit_count(ref) + request = Gitaly::CountCommitsRequest.new( + repository: @gitaly_repo, + revision: ref + ) + + GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count + end + + def between(from, to) + request = Gitaly::CommitsBetweenRequest.new( + repository: @gitaly_repo, + from: from, + to: to + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + consume_commits_response(response) + end + + def find_all_commits(opts = {}) + request = Gitaly::FindAllCommitsRequest.new( + repository: @gitaly_repo, + revision: opts[:ref].to_s, + max_count: opts[:max_count].to_i, + skip: opts[:skip].to_i + ) + request.order = opts[:order].upcase if opts[:order].present? + + response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request) + consume_commits_response(response) + end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def consume_commits_response(response) + response.flat_map do |message| + message.commits.map do |gitaly_commit| + commit = GitalyClient::Commit.new(@repository, gitaly_commit) + Gitlab::Git::Commit.new(commit) + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index 1e117b7e74a..d459c9a88fb 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,7 +1,7 @@ module Gitlab module GitalyClient class Diff - FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze attr_accessor(*FIELDS) diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index d84e8d752dc..65d81dc5d46 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -13,7 +13,10 @@ module Gitlab @rpc_response.each do |diff_msg| if current_diff.nil? diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) - diff_params[:patch] = diff_msg.raw_patch_data + # gRPC uses frozen strings by default, and we need to have an unfrozen string as it + # gets processed further down the line. So we unfreeze the first chunk of the patch + # in case it's the only chunk we receive for this diff. + diff_params[:patch] = diff_msg.raw_patch_data.dup current_diff = GitalyClient::Diff.new(diff_params) else diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notification_service.rb index 78ed433e6b8..326e6f7dafc 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Notifications + class NotificationService # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository @@ -10,7 +10,7 @@ module Gitlab def post_receive GitalyClient.call( @storage, - :notifications, + :notification_service, :post_receive, Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) ) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb deleted file mode 100644 index 6d5f54dd959..00000000000 --- a/lib/gitlab/gitaly_client/ref.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Gitlab - module GitalyClient - class Ref - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @storage = repository.storage - end - - def default_branch_name - request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request) - Gitlab::Git.branch_name(response.name) - end - - def branch_names - request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request) - consume_refs_response(response, prefix: 'refs/heads/') - end - - def tag_names - request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request) - consume_refs_response(response, prefix: 'refs/tags/') - end - - def find_ref_name(commit_id, ref_prefix) - request = Gitaly::FindRefNameRequest.new( - repository: @gitaly_repo, - commit_id: commit_id, - prefix: ref_prefix - ) - GitalyClient.call(@storage, :ref, :find_ref_name, request).name - end - - def count_tag_names - tag_names.count - end - - def count_branch_names - branch_names.count - end - - def local_branches(sort_by: nil) - request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) - request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref, :find_local_branches, request) - consume_branches_response(response) - end - - private - - def consume_refs_response(response, prefix:) - response.flat_map do |r| - r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } - end - end - - def sort_by_param(sort_by) - enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) - raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value - enum_value - end - - def consume_branches_response(response) - response.flat_map { |r| r.branches } - end - end - end -end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb new file mode 100644 index 00000000000..b0f7548b7dc --- /dev/null +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -0,0 +1,148 @@ +module Gitlab + module GitalyClient + class RefService + include Gitlab::EncodingHelper + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def branches + request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + response.flat_map do |message| + message.branches.map do |branch| + gitaly_commit = GitalyClient::Commit.new(@repository, branch.target) + target_commit = Gitlab::Git::Commit.decorate(gitaly_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + + def default_branch_name + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) + Gitlab::Git.branch_name(response.name) + end + + def branch_names + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) + consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } + end + + def tag_names + request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) + consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } + end + + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @gitaly_repo, + commit_id: commit_id, + prefix: ref_prefix + ) + encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) + end + + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) + consume_branches_response(response) + end + + def tags + request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + consume_tags_response(response) + end + + private + + def consume_refs_response(response) + response.flat_map { |message| message.names.map { |name| yield(name) } } + end + + def sort_by_param(sort_by) + sort_by = 'name' if sort_by == 'name_asc' + + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map do |message| + message.branches.map do |gitaly_branch| + Gitlab::Git::Branch.new( + @repository, + encode!(gitaly_branch.name.dup), + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) + ) + end + end + end + + def consume_tags_response(response) + response.flat_map do |message| + message.tags.map do |gitaly_tag| + if gitaly_tag.target_commit.present? + commit = GitalyClient::Commit.new(@repository, gitaly_tag.target_commit) + gitaly_commit = Gitlab::Git::Commit.new(commit) + end + + Gitlab::Git::Tag.new( + @repository, + encode!(gitaly_tag.name.dup), + gitaly_tag.id, + gitaly_commit, + encode!(gitaly_tag.message.chomp) + ) + end + end + end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name.dup, + author_email: response.commit_author.email.dup, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name.dup, + committer_email: response.commit_committer.email.dup + } + + Gitlab::Git::Commit.decorate(hash) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb new file mode 100644 index 00000000000..13e75b256a7 --- /dev/null +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -0,0 +1,32 @@ +module Gitlab + module GitalyClient + class RepositoryService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def exists? + request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :repository_service, :exists, request).exists + end + + def garbage_collect(create_bitmap) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request) + end + + def repack_full(create_bitmap) + request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + GitalyClient.call(@storage, :repository_service, :repack_full, request) + end + + def repack_incremental + request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request) + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 319633656ff..2d1ae6a5925 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -2,11 +2,14 @@ module Gitlab module GonHelper + include WebpackHelper + def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb new file mode 100644 index 00000000000..e1d1724295a --- /dev/null +++ b/lib/gitlab/gpg.rb @@ -0,0 +1,62 @@ +module Gitlab + module Gpg + extend self + + module CurrentKeyChain + extend self + + def add(key) + GPGME::Key.import(key) + end + + def fingerprints_from_key(key) + import = GPGME::Key.import(key) + + return [] if import.imported == 0 + + import.imports.map(&:fingerprint) + end + end + + def fingerprints_from_key(key) + using_tmp_keychain do + CurrentKeyChain.fingerprints_from_key(key) + end + end + + def primary_keyids_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).map { |raw_key| raw_key.primary_subkey.keyid } + end + end + + def user_infos_from_key(key) + using_tmp_keychain do + fingerprints = CurrentKeyChain.fingerprints_from_key(key) + + GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| + raw_key.uids.map { |uid| { name: uid.name, email: uid.email } } + end + end + end + + def using_tmp_keychain + Dir.mktmpdir do |dir| + @original_dirs ||= [GPGME::Engine.dirinfo('homedir')] + @original_dirs.push(dir) + + GPGME::Engine.home_dir = dir + + return_value = yield + + @original_dirs.pop + + GPGME::Engine.home_dir = @original_dirs[-1] + + return_value + end + end + end +end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb new file mode 100644 index 00000000000..55428b85207 --- /dev/null +++ b/lib/gitlab/gpg/commit.rb @@ -0,0 +1,85 @@ +module Gitlab + module Gpg + class Commit + attr_reader :commit + + def initialize(commit) + @commit = commit + + @signature_text, @signed_text = commit.raw.signature(commit.project.repository) + end + + def has_signature? + !!(@signature_text && @signed_text) + end + + def signature + return unless has_signature? + + cached_signature = GpgSignature.find_by(commit_sha: commit.sha) + return cached_signature if cached_signature.present? + + using_keychain do |gpg_key| + create_cached_signature!(gpg_key) + end + end + + def update_signature!(cached_signature) + using_keychain do |gpg_key| + cached_signature.update_attributes!(attributes(gpg_key)) + end + end + + private + + def using_keychain + Gitlab::Gpg.using_tmp_keychain do + # first we need to get the keyid from the signature to query the gpg + # key belonging to the keyid. + # This way we can add the key to the temporary keychain and extract + # the proper signature. + gpg_key = GpgKey.find_by(primary_keyid: verified_signature.fingerprint) + + if gpg_key + Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) + @verified_signature = nil + end + + yield gpg_key + end + end + + def verified_signature + @verified_signature ||= GPGME::Crypto.new.verify(@signature_text, signed_text: @signed_text) do |verified_signature| + break verified_signature + end + end + + def create_cached_signature!(gpg_key) + GpgSignature.create!(attributes(gpg_key)) + end + + def attributes(gpg_key) + user_infos = user_infos(gpg_key) + + { + commit_sha: commit.sha, + project: commit.project, + gpg_key: gpg_key, + gpg_key_primary_keyid: gpg_key&.primary_keyid || verified_signature.fingerprint, + gpg_key_user_name: user_infos[:name], + gpg_key_user_email: user_infos[:email], + valid_signature: gpg_signature_valid_signature_value(gpg_key) + } + end + + def gpg_signature_valid_signature_value(gpg_key) + !!(gpg_key && gpg_key.verified? && verified_signature.valid?) + end + + def user_infos(gpg_key) + gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} + end + end + end +end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb new file mode 100644 index 00000000000..3bb491120ba --- /dev/null +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -0,0 +1,19 @@ +module Gitlab + module Gpg + class InvalidGpgSignatureUpdater + def initialize(gpg_key) + @gpg_key = gpg_key + end + + def run + GpgSignature + .select(:id, :commit_sha, :project_id) + .where('gpg_key_id IS NULL OR valid_signature = ?', false) + .where(gpg_key_primary_keyid: @gpg_key.primary_keyid) + .find_each do |gpg_signature| + Gitlab::Gpg::Commit.new(gpg_signature.commit).update_signature!(gpg_signature) + end + end + end + end +end diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7de6d4d9367..8b365dab185 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -27,10 +27,10 @@ module Gitlab Metric.new(name, value, labels) end - def with_timing(proc) + def with_timing start = Time.now - result = proc.call - yield result, Time.now.to_f - start.to_f + result = yield + [result, Time.now.to_f - start.to_f] end def catch_timeout(seconds, &block) diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index e78b7f22e03..9e91c135956 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,49 +10,47 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - tmp_file_path = tmp_file_path(storage_name) - if !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - elsif !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) else - HealthChecks::Result.new(true, nil, shard: storage_name) + with_temp_file(storage_name) do |tmp_file_path| + if !storage_write_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) + elsif !storage_read_test(tmp_file_path) + HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) + else + HealthChecks::Result.new(true, nil, shard: storage_name) + end + end end rescue RuntimeError => ex message = "unexpected error #{ex} when checking storage #{storage_name}" Rails.logger.error(message) HealthChecks::Result.new(false, message, shard: storage_name) - ensure - delete_test_file(tmp_file_path) end end end def metrics repository_storages.flat_map do |storage_name| - tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + storage_stat_metrics(storage_name), + storage_write_metrics(storage_name), + storage_read_metrics(storage_name) ].flatten end end private - def operation_metrics(ok_metric, latency_metric, operation, **labels) - with_timing operation do |result, elapsed| - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - end + def operation_metrics(ok_metric, latency_metric, **labels) + result, elapsed = yield + [ + metric(latency_metric, elapsed, **labels), + metric(ok_metric, result ? 1 : 0, **labels) + ] rescue RuntimeError => ex - Rails.logger("unexpected error #{ex} when checking #{ok_metric}") + Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") [metric(ok_metric, 0, **labels)] end @@ -68,19 +66,36 @@ module Gitlab Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end - def tmp_file_path(storage_name) - Dir::Tmpname.create(%w(fs_shards_check +deleted), path(storage_name)) { |path| path } + def with_temp_file(storage_name) + temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } + yield temp_file_path + ensure + delete_test_file(temp_file_path) end - def path(storage_name) + def storage_path(storage_name) storages_paths&.dig(storage_name, 'path') end + # All below test methods use shell commands to perform actions on storage volumes. + # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, + # we can rely on shell commands to be terminated once `timeout` kills them. + # + # However we also fallback to pure Ruby file operations in case a specific shell command is missing + # so we are still able to perform healthchecks and gather metrics from such system. + + def delete_test_file(tmp_path) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) + status.zero? + rescue Errno::ENOENT + File.delete(tmp_path) rescue Errno::ENOENT + end + def storage_stat_test(storage_name) - stat_path = File.join(path(storage_name), '.') + stat_path = File.join(storage_path(storage_name), '.') begin _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status == 0 + status.zero? rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? end @@ -90,7 +105,7 @@ module Gitlab _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT written_bytes == RANDOM_STRING.length @@ -100,17 +115,33 @@ module Gitlab _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end - status == 0 + status.zero? rescue Errno::ENOENT file_contents = File.read(tmp_path) rescue Errno::ENOENT file_contents == RANDOM_STRING end - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status == 0 - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT + def storage_stat_metrics(storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do + with_timing { storage_stat_test(storage_name) } + end + end + + def storage_write_metrics(storage_name) + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + with_timing { storage_write_test(tmp_file_path) } + end + end + end + + def storage_read_metrics(storage_name) + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do + with_temp_file(storage_name) do |tmp_file_path| + storage_write_test(tmp_file_path) # writes data used by read test + with_timing { storage_read_test(tmp_file_path) } + end + end end end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..f5026171ba4 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -15,14 +15,13 @@ module Gitlab end def metrics - with_timing method(:check) do |result, elapsed| - Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) - [ - metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), - metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) - ] - end + result, elapsed = with_timing(&method(:check)) + Rails.logger.error("#{human_name} check returned unexpected result #{result}") unless is_successful?(result) + [ + metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), + metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), + metric("#{metric_prefix}_latency_seconds", elapsed) + ] end private diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index a5ad2f952d3..cc282d1415b 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -7,11 +7,16 @@ module Gitlab 'es' => 'Español', 'de' => 'Deutsch', 'fr' => 'Français', - 'pt_BR' => 'Português(Brasil)', + 'pt_BR' => 'Português (Brasil)', 'zh_CN' => '简体中文', - 'zh_HK' => '繁體中文(香港)', - 'zh_TW' => '繁體中文(臺灣)', - 'bg' => 'български' + 'zh_HK' => '繁體中文 (香港)', + 'zh_TW' => '繁體中文 (臺灣)', + 'bg' => 'български', + 'ru' => 'Русский', + 'eo' => 'Esperanto', + 'it' => 'Italiano', + 'uk' => 'Українська', + 'ja' => '日本語' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 72183e8aad4..c8ad3a7a5e0 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,7 @@ project_tree: - :author - :events - merge_request_diff: + - :merge_request_diff_commits - :merge_request_diff_files - :events - :timelogs @@ -99,6 +100,7 @@ excluded_attributes: - :milestone_id merge_requests: - :milestone_id + - :ref_fetched award_emoji: - :awardable_id statuses: diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index c56c1a4322f..cdbdfa10d0e 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -76,5 +76,44 @@ module Gitlab url.to_s end + + def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + config = { + apiVersion: 'v1', + clusters: [ + name: 'gitlab-deploy', + cluster: { + server: url + } + ], + contexts: [ + name: 'gitlab-deploy', + context: { + cluster: 'gitlab-deploy', + namespace: namespace, + user: 'gitlab-deploy' + } + ], + 'current-context': 'gitlab-deploy', + kind: 'Config', + users: [ + { + name: 'gitlab-deploy', + user: { token: token } + } + ] + } + + kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem + + config.deep_stringify_keys + end + + private + + def kubeconfig_embed_ca_pem(config, ca_pem) + cluster = config.dig(:clusters, 0, :cluster) + cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 54a5b1d31cd..fb68627dedf 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -16,8 +16,8 @@ module Gitlab def self.allowed?(user) self.open(user) do |access| if access.allowed? - user.last_credential_check_at = Time.now - user.save + Users::UpdateService.new(user, last_credential_check_at: Time.now).execute + true else false diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 7b05290e5cc..8867a91c244 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -101,7 +101,7 @@ module Gitlab end def user_attributes - %W(#{config.uid} cn mail dn) + %W(#{config.uid} cn dn) + config.attributes['username'] + config.attributes['email'] end end end diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index 4745311402c..ed1de73f8c6 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -42,7 +42,7 @@ module Gitlab end def adapter - OmniAuth::LDAP::Adaptor.new(config.options.symbolize_keys) + OmniAuth::LDAP::Adaptor.new(config.omniauth_options) end def config diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 6fdf68641e2..c8f19cd52d5 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -2,6 +2,12 @@ module Gitlab module LDAP class Config + NET_LDAP_ENCRYPTION_METHOD = { + simple_tls: :simple_tls, + start_tls: :start_tls, + plain: nil + }.freeze + attr_accessor :provider, :options def self.enabled? @@ -12,6 +18,12 @@ module Gitlab Gitlab.config.ldap.servers.values end + def self.available_servers + return [] unless enabled? + + Array.wrap(servers.first) + end + def self.providers servers.map { |server| server['provider_name'] } end @@ -39,7 +51,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption + encryption: encryption_options ) opts.merge!(auth_options) if has_auth? @@ -50,9 +62,10 @@ module Gitlab def omniauth_options opts = base_options.merge( base: base, - method: options['method'], + encryption: options['encryption'], filter: omniauth_user_filter, - name_proc: name_proc + name_proc: name_proc, + disable_verify_certificates: !options['verify_certificates'] ) if has_auth? @@ -62,6 +75,9 @@ module Gitlab ) end + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + opts end @@ -157,15 +173,37 @@ module Gitlab base_config.servers.values.find { |server| server['provider_name'] == provider } end - def encryption - case options['method'].to_s - when 'ssl' - :simple_tls - when 'tls' - :start_tls - else - nil - end + def encryption_options + method = translate_method(options['encryption']) + return nil unless method + + { + method: method, + tls_options: tls_options(method) + } + end + + def translate_method(method_from_config) + NET_LDAP_ENCRYPTION_METHOD[method_from_config.to_sym] + end + + def tls_options(method) + return { verify_mode: OpenSSL::SSL::VERIFY_NONE } unless method + + opts = if options['verify_certificates'] + OpenSSL::SSL::SSLContext::DEFAULT_PARAMS + else + # It is important to explicitly set verify_mode for two reasons: + # 1. The behavior of OpenSSL is undefined when verify_mode is not set. + # 2. The net-ldap gem implementation verifies the certificate hostname + # unless verify_mode is set to VERIFY_NONE. + { verify_mode: OpenSSL::SSL::VERIFY_NONE } + end + + opts[:ca_file] = options['ca_file'] if options['ca_file'].present? + opts[:ssl_version] = options['ssl_version'] if options['ssl_version'].present? + + opts end def auth_options diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb new file mode 100644 index 00000000000..219accfc029 --- /dev/null +++ b/lib/gitlab/metrics/base_sampler.rb @@ -0,0 +1,94 @@ +require 'logger' +module Gitlab + module Metrics + class BaseSampler + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance + end + + attr_reader :running + + # interval - The sampling interval in seconds. + def initialize(interval) + interval_half = interval.to_f / 2 + + @interval = interval + @interval_steps = (-interval_half..interval_half).step(0.1).to_a + + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return if running + @running = true + + @thread = Thread.new do + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + end + end + + def stop + @mutex.synchronize do + return unless running + + @running = false + + if @thread + @thread.wakeup if @thread.alive? + @thread.join + @thread = nil + end + end + end + + def safe_sample + sample + rescue => e + Rails.logger.warn("#{self.class}: #{e}, stopping") + stop + end + + def sample + raise NotImplementedError + end + + # Returns the sleep interval with a random adjustment. + # + # The random adjustment is put in place to ensure we: + # + # 1. Don't generate samples at the exact same interval every time (thus + # potentially missing anything that happens in between samples). + # 2. Don't sample data at the same interval two times in a row. + def sleep_interval + while step = @interval_steps.sample + if step != @last_step + @last_step = step + + return @interval + @last_step + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/influx_sampler.rb index 0000450d9bb..6db1dd755b7 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/influx_sampler.rb @@ -5,14 +5,11 @@ module Gitlab # This class is used to gather statistics that can't be directly associated # with a transaction such as system memory usage, garbage collection # statistics, etc. - class Sampler + class InfluxSampler < BaseSampler # interval - The sampling interval in seconds. def initialize(interval = Metrics.settings[:sample_interval]) - interval_half = interval.to_f / 2 - - @interval = interval - @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @last_step = nil + super(interval) + @last_step = nil @metrics = [] @@ -26,18 +23,6 @@ module Gitlab end end - def start - Thread.new do - Thread.current.abort_on_exception = true - - loop do - sleep(sleep_interval) - - sample - end - end - end - def sample sample_memory_usage sample_file_descriptors @@ -86,7 +71,7 @@ module Gitlab end def sample_gc - time = GC::Profiler.total_time * 1000.0 + time = GC::Profiler.total_time * 1000.0 stats = GC.stat.merge(total_time: time) # We want the difference of GC runs compared to the last sample, not the @@ -111,23 +96,6 @@ module Gitlab def sidekiq? Sidekiq.server? end - - # Returns the sleep interval with a random adjustment. - # - # The random adjustment is put in place to ensure we: - # - # 1. Don't generate samples at the exact same interval every time (thus - # potentially missing anything that happens in between samples). - # 2. Don't sample data at the same interval two times in a row. - def sleep_interval - while step = @interval_steps.sample - if step != @last_step - @last_step = step - - return @interval + @last_step - end - end - end end end end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 9d314a56e58..460dab47276 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -6,9 +6,11 @@ module Gitlab include Gitlab::CurrentSettings def metrics_folder_present? - ENV.has_key?('prometheus_multiproc_dir') && - ::Dir.exist?(ENV['prometheus_multiproc_dir']) && - ::File.writable?(ENV['prometheus_multiproc_dir']) + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) end def prometheus_metrics_enabled? @@ -29,8 +31,8 @@ module Gitlab provide_metric(name) || registry.summary(name, docstring, base_labels) end - def gauge(name, docstring, base_labels = {}) - provide_metric(name) || registry.gauge(name, docstring, base_labels) + def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) + provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode) end def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb new file mode 100644 index 00000000000..f6987252039 --- /dev/null +++ b/lib/gitlab/metrics/unicorn_sampler.rb @@ -0,0 +1,48 @@ +module Gitlab + module Metrics + class UnicornSampler < BaseSampler + def initialize(interval) + super(interval) + end + + def unicorn_active_connections + @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max) + end + + def unicorn_queued_connections + @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max) + end + + def enabled? + # Raindrops::Linux.tcp_listener_stats is only present on Linux + unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats) + end + + def sample + Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued) + end + + Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats| + unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active) + unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued) + end + end + + private + + def tcp_listeners + @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z}) + end + + def unix_listeners + @unix_listeners ||= Unicorn.listener_names - tcp_listeners + end + + def unicorn_with_listeners? + defined?(Unicorn) && Unicorn.listener_names.any? + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 7307f8c2c87..3f2bbd9f6a6 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -32,7 +32,7 @@ module Gitlab block_after_save = needs_blocking? - gl_user.save! + Users::UpdateService.new(gl_user).execute! gl_user.block if block_after_save @@ -101,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 0d541935bc6..22332474945 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -34,7 +34,7 @@ module Gitlab write_csv do |csv| ActiveRecord::Base.transaction do - User.with_two_factor.in_batches do |relation| + User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) rows.each do |row| user = %i[id ciphertext iv salt].zip(row).to_h diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 10eb99fb461..894bd5efae5 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -14,43 +14,42 @@ module Gitlab TOP_LEVEL_ROUTES = %w[ - .well-known + 404.html + 422.html + 500.html + 502.html + 503.html abuse_reports admin - all api + apple-touch-icon-precomposed.png + apple-touch-icon.png assets autocomplete ci dashboard + deploy.html explore + favicon.ico files groups health_check help - hooks import invites - issues jwt koding - member - merge_requests - new - notes notification_settings oauth profile projects public - repository robots.txt s search sent_notifications - services + slash-command-logo.png snippets - system - teams u unicorn_test unsubscribes @@ -112,6 +111,7 @@ module Gitlab # this group would not be accessible through `/groups/parent/activity` since # this would map to the activity-page of its parent. GROUP_ROUTES = %w[ + - activity analytics audit_events diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 163a40ad306..56112ec2301 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,34 @@ module Gitlab module PerformanceBar - def self.enabled? - Feature.enabled?('gitlab_performance_bar') + include Gitlab::CurrentSettings + + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes + + def self.enabled?(user = nil) + return false unless user && allowed_group_id + + allowed_user_ids.include?(user.id) + end + + def self.allowed_group_id + current_application_settings.performance_bar_allowed_group_id + end + + def self.allowed_user_ids + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do + group = Group.find_by_id(allowed_group_id) + + if group + GroupMembersFinder.new(group).execute.pluck(:user_id) + else + [] + end + end + end + + def self.expire_allowed_user_ids_cache + Rails.cache.delete(ALLOWED_USER_IDS_KEY) end end end diff --git a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb deleted file mode 100644 index d939a6ea18d..00000000000 --- a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This solves a bug with a X-Senfile header that wouldn't be set properly, see -# https://github.com/peek/peek-performance_bar/pull/27 -module Gitlab - module PerformanceBar - module PeekPerformanceBarWithRackBody - def call(env) - @env = env - reset_stats - - @total_requests += 1 - first_request if @total_requests == 1 - - env['process.request_start'] = @start.to_f - env['process.total_requests'] = total_requests - - status, headers, body = @app.call(env) - body = Rack::BodyProxy.new(body) { record_request } - [status, headers, body] - end - end - end -end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 574ae8731a5..67fee8c227d 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -1,4 +1,5 @@ # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb +# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb module Gitlab module PerformanceBar module PeekQueryTracker @@ -23,14 +24,20 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if RequestStore.active? && RequestStore.store[:peek_enabled] - track_query(data[:sql].strip, data[:binds], start, finish) + # data[:cached] is only available starting from Rails 5.1.0 + # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 + # Before that, data[:name] was set to 'CACHE' + # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end end end end def track_query(raw_query, bindings, start, finish) query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: '%.3f' % query.duration, sql: query.formatted_query } + query_info = { duration: query.duration.round(3), sql: query.formatted_query } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb new file mode 100644 index 00000000000..cb95daf2260 --- /dev/null +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Prometheus + module AdditionalMetricsParser + extend self + + def load_groups_from_yaml + additional_metrics_raw.map(&method(:group_from_entry)) + end + + private + + def validate!(obj) + raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid? + end + + def group_from_entry(entry) + entry[:name] = entry.delete(:group) + entry[:metrics]&.map! do |entry| + Metric.new(entry).tap(&method(:validate!)) + end + + MetricGroup.new(entry).tap(&method(:validate!)) + end + + def additional_metrics_raw + load_yaml_file&.map(&:deep_symbolize_keys).freeze + end + + def load_yaml_file + @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + end + end + end +end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb new file mode 100644 index 00000000000..f54b2c6aaff --- /dev/null +++ b/lib/gitlab/prometheus/metric.rb @@ -0,0 +1,16 @@ +module Gitlab + module Prometheus + class Metric + include ActiveModel::Model + + attr_accessor :title, :required_metrics, :weight, :y_label, :queries + + validates :title, :required_metrics, :weight, :y_label, :queries, presence: true + + def initialize(params = {}) + super(params) + @y_label ||= 'Values' + end + end + end +end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb new file mode 100644 index 00000000000..729fef34b35 --- /dev/null +++ b/lib/gitlab/prometheus/metric_group.rb @@ -0,0 +1,14 @@ +module Gitlab + module Prometheus + class MetricGroup + include ActiveModel::Model + + attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true + + def self.all + AdditionalMetricsParser.load_groups_from_yaml + end + end + end +end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb new file mode 100644 index 00000000000..49cc0e16080 --- /dev/null +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -0,0 +1,5 @@ +module Gitlab + module Prometheus + ParsingError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb new file mode 100644 index 00000000000..67c69d9ccf3 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -0,0 +1,22 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsDeploymentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + query_context = { + environment_slug: deployment.environment.slug, + environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + } + + query_metrics(query_context) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb new file mode 100644 index 00000000000..b5a679ddd79 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -0,0 +1,22 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsEnvironmentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + query_context = { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + timeframe_start: 8.hours.ago.to_f, + timeframe_end: Time.now.to_f + } + + query_metrics(query_context) + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 2a2eb4ae57f..c60828165bd 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client - delegate :query_range, :query, to: :client, prefix: true + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 2cc08731f8d..170f483540e 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,26 +1,31 @@ -module Gitlab::Prometheus::Queries - class DeploymentQuery < BaseQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - environment_slug = deployment.environment.slug +module Gitlab + module Prometheus + module Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + Deployment.find_by(id: deployment_id).try do |deployment| + environment_slug = deployment.environment.slug - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 01d756d7284..66f29d95177 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,20 +1,25 @@ -module Gitlab::Prometheus::Queries - class EnvironmentQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f +module Gitlab + module Prometheus + module Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + Environment.find_by(id: environment_id).try do |environment| + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb new file mode 100644 index 00000000000..d4894c87f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -0,0 +1,80 @@ +module Gitlab + module Prometheus + module Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze + + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end + + private + + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) + + groups = {} + + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end + + groups + end + + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now + + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end + + def has_matching_label(series_info) + series_info.key?('environment') + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end + + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end + + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb new file mode 100644 index 00000000000..e44be770544 --- /dev/null +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -0,0 +1,73 @@ +module Gitlab + module Prometheus + module Queries + module QueryAdditionalMetrics + def query_metrics(query_context) + query_processor = method(:process_query).curry[query_context] + + groups = matched_metrics.map do |group| + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + y_label: metric.y_label, + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) + } + end + + { + group: group.name, + priority: group.priority, + metrics: metrics.select(&method(:metric_with_any_queries)) + } + end + + groups.select(&method(:group_with_any_metrics)) + end + + private + + def metric_with_any_queries(metric) + metric[:queries]&.count&.> 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count&.> 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_query(context, query) + query_with_result = query.dup + result = + if query.key?(:query_range) + client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + else + client_query(query[:query] % context, time: context[:timeframe_end]) + end + query_with_result[:result] = result&.map(&:deep_symbolize_keys) + query_with_result + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def matched_metrics + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + group + end + + result.select { |group| group.metrics.any? } + end + end + end + end +end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 5b51a1779dd..aa94614bf18 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -29,6 +29,14 @@ module Gitlab end end + def label_values(name = '__name__') + json_api_get("label/#{name}/values") + end + + def series(*matches, start: 8.hours.ago, stop: Time.now) + json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) + end + private def json_api_get(type, args = {}) diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index a4a97236ffc..536765305e1 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -105,9 +105,32 @@ module Gitlab # # Awesome code block # end def command(*command_names, &block) + define_command(CommandDefinition, *command_names, &block) + end + + # Registers a new substitution which is recognizable from body of email or + # comment. + # It accepts aliases and takes a block with the formatted content. + # + # Example: + # + # command :my_substitution, :alias_for_my_substitution do |text| + # "#{text} MY AWESOME SUBSTITUTION" + # end + def substitution(*substitution_names, &block) + define_command(SubstitutionDefinition, *substitution_names, &block) + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] + end + + private + + def define_command(klass, *command_names, &block) name, *aliases = command_names - definition = CommandDefinition.new( + definition = klass.new( name, aliases: aliases, description: @description, @@ -130,10 +153,6 @@ module Gitlab @condition_block = nil @parse_params_block = nil end - - def definition_by_name(name) - command_definitions_by_name[name.to_sym] - end end end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 09576be7156..3ebfa3bd4b8 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -46,6 +46,8 @@ module Gitlab end end + content, commands = perform_substitutions(content, commands) + [content.strip, commands] end @@ -110,6 +112,26 @@ module Gitlab }mx end + def perform_substitutions(content, commands) + return unless content + + substitution_definitions = self.command_definitions.select do |definition| + definition.is_a?(Gitlab::QuickActions::SubstitutionDefinition) + end + + substitution_definitions.each do |substitution| + match_data = substitution.match(content) + if match_data + command = [substitution.name.to_s] + command << match_data[1] unless match_data[1].empty? + commands << command + end + content = substitution.perform_substitution(self, content) + end + + [content, commands] + end + def command_names(opts) command_definitions.flat_map do |command| next if command.noop? diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb new file mode 100644 index 00000000000..032c49ed159 --- /dev/null +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -0,0 +1,24 @@ +module Gitlab + module QuickActions + class SubstitutionDefinition < CommandDefinition + # noop?=>true means these won't get extracted or removed by Gitlab::QuickActions::Extractor#extract_commands + # QuickActions::InterpretService#perform_substitutions handles them separately + def noop? + true + end + + def match(content) + content.match %r{^/#{all_names.join('|')} ?(.*)$} + end + + def perform_substitution(context, content) + return unless content + + all_names.each do |a_name| + content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + end + content + end + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 7668ecacc4b..f5b757ace77 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -33,7 +33,12 @@ module Gitlab def issues if project && project.jira_tracker? - @references[:external_issue] ||= references(:external_issue) + if project.issues_enabled? + @references[:all_issues] ||= references(:external_issue) + references(:issue) + else + @references[:external_issue] ||= references(:external_issue) + + references(:issue).select { |i| i.project_id != project.id } + end else @references[:issue] ||= references(:issue) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b706434217d..1adc5ec952a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -19,27 +19,29 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def file_name_regex - @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze - end - - def file_name_regex_message - "can contain only letters, digits, '_', '-', '@', '+' and '.'." - end - - def container_registry_reference_regex - Gitlab::PathRegex.git_reference_regex - end - ## - # Docker Distribution Registry 2.4.1 repository name rules + # Docker Distribution Registry repository / tag name rules + # + # See https://github.com/docker/distribution/blob/master/reference/regexp.go. # def container_repository_name_regex @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} end + ## + # We do not use regexp anchors here because these are not allowed when + # used as a routing constraint. + # + def container_registry_tag_regex + @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/ + end + + def environment_name_regex_chars + 'a-zA-Z0-9_/\\$\\{\\}\\. -' + end + def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars}]+\z/.freeze end def environment_name_regex_message diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb new file mode 100644 index 00000000000..ccfe0d6bed3 --- /dev/null +++ b/lib/gitlab/request_forgery_protection.rb @@ -0,0 +1,39 @@ +# A module to check CSRF tokens in requests. +# It's used in API helpers and OmniAuth. +# Usage: GitLab::RequestForgeryProtection.call(env) + +module Gitlab + module RequestForgeryProtection + class Controller < ActionController::Base + protect_from_forgery with: :exception + + rescue_from ActionController::InvalidAuthenticityToken do |e| + logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" + logger.warn "Unlike the logs may suggest, this does not result in an actual 422 response to the user" + logger.warn "For API requests, the only effect is that `current_user` will be `nil` for the duration of the request" + + raise e + end + + def index + head :ok + end + end + + def self.app + @app ||= Controller.action(:index) + end + + def self.call(env) + app.call(env) + end + + def self.verified?(env) + call(env) + + true + rescue ActionController::InvalidAuthenticityToken + false + end + end +end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 877aa6e6a28..f3952657983 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -18,7 +18,11 @@ module Gitlab mapping = @map.find { |mapping| mapping[:source] === path } return unless mapping - path.sub(mapping[:source], mapping[:public]) + if mapping[:source].is_a?(String) + path.sub(mapping[:source], mapping[:public]) + else + mapping[:source].replace(path, mapping[:public]) + end end private @@ -35,7 +39,7 @@ module Gitlab source_pattern = source_pattern[1...-1].gsub('\/', '/') begin - source_pattern = /\A#{source_pattern}\z/ + source_pattern = Gitlab::UntrustedRegexp.new('\A' + source_pattern + '\z') rescue RegexpError => e raise FormatError, "Route map entry source is not a valid regular expression: #{e}" end diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb deleted file mode 100644 index 36d1a8a6f64..00000000000 --- a/lib/gitlab/routes/legacy_builds.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module Routes - class LegacyBuilds - def initialize(map) - @map = map - end - - def draw - @map.instance_eval do - resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do - collection do - resources :artifacts, only: [], controller: 'build_artifacts' do - collection do - get :latest_succeeded, - path: '*ref_name_and_path', - format: false - end - end - end - - member do - get :raw - end - - resource :artifacts, only: [], controller: 'build_artifacts' do - get :download - get :browse, path: 'browse(/*path)', format: false - get :file, path: 'file/*path', format: false - get :raw, path: 'raw/*path', format: false - end - end - end - end - end - end -end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 632e2d87500..e57890f1143 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -2,10 +2,35 @@ module Gitlab module Routing extend ActiveSupport::Concern + mattr_accessor :_includers + self._includers = [] + included do + Gitlab::Routing.includes_helpers(self) + include Gitlab::Routing.url_helpers end + def self.includes_helpers(klass) + self._includers << klass + end + + def self.add_helpers(mod) + url_helpers.include mod + url_helpers.extend mod + + GitlabRoutingHelper.include mod + GitlabRoutingHelper.extend mod + + app_url_helpers = Gitlab::Application.routes.named_routes.url_helpers_module + app_url_helpers.include mod + app_url_helpers.extend mod + + _includers.each do |klass| + klass.include mod + end + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 22554236c38..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,7 +1,12 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab class Shell + GITLAB_SHELL_ENV_VARS = %w(GIT_TERMINAL_PROMPT).freeze + Error = Class.new(StandardError) KeyAdder = Struct.new(:io) do @@ -66,9 +71,10 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'add-project', storage, "#{name}.git"]) end # Import repository @@ -79,13 +85,13 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. - output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) - raise Error, output unless status.zero? - true + cmd = [gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"] + gitlab_shell_fast_execute_raise_error(cmd) end # Fetch remote for repository @@ -98,14 +104,13 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced args << '--no-tags' if no_tags - output, status = Popen.popen(args) - raise Error, output unless status.zero? - true + gitlab_shell_fast_execute_raise_error(args) end # Move repository @@ -116,9 +121,10 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - storage, "#{path}.git", "#{new_path}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace @@ -130,10 +136,11 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - forked_from_storage, "#{path}.git", forked_to_storage, - fork_namespace]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', + forked_from_storage, "#{path}.git", forked_to_storage, + fork_namespace]) end # Remove repository from file system @@ -144,9 +151,10 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', storage, "#{name}.git"]) + gitlab_shell_fast_execute([gitlab_shell_projects_path, + 'rm-project', storage, "#{name}.git"]) end # Add new key to gitlab-shell @@ -155,8 +163,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, self.class.strip_key(key_content)]) + gitlab_shell_fast_execute([gitlab_shell_keys_path, + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -175,8 +183,10 @@ module Gitlab # remove_key("key-342", "sha-rsa ...") # def remove_key(key_id, key_content) - Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'rm-key', key_id, key_content]) + args = [gitlab_shell_keys_path, 'rm-key', key_id] + args << key_content if key_content + + gitlab_shell_fast_execute(args) end # Remove all ssh keys from gitlab shell @@ -185,7 +195,7 @@ module Gitlab # remove_all_keys # def remove_all_keys - Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear']) + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end # Add empty directory for storing repositories @@ -193,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -206,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -215,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -240,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end @@ -267,5 +281,31 @@ module Gitlab def gitlab_shell_keys_path File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + + private + + def gitlab_shell_fast_execute(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + return true if status.zero? + + Rails.logger.error("gitlab-shell failed with error #{status}: #{output}") + false + end + + def gitlab_shell_fast_execute_raise_error(cmd) + output, status = gitlab_shell_fast_execute_helper(cmd) + + raise Error, output unless status.zero? + true + end + + def gitlab_shell_fast_execute_helper(cmd) + vars = ENV.to_h.slice(*GITLAB_SHELL_ENV_VARS) + + # Don't pass along the entire parent environment to prevent gitlab-shell + # from wasting I/O by searching through GEM_PATH + Bundler.with_original_env { Popen.popen(cmd, nil, vars) } + end end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index e71eb15d604..93e00ab75a1 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -21,29 +21,34 @@ module Gitlab from = match[:from] to = match[:to] - actions = find_actions(from, to) + action = find_action(from, to) - if actions.none? - Gitlab::SlashCommands::Presenters::Deploy.new(nil).no_actions - elsif actions.one? - action = play!(from, to, actions.first) - Gitlab::SlashCommands::Presenters::Deploy.new(action).present(from, to) + if action.nil? + Gitlab::SlashCommands::Presenters::Deploy + .new(action).action_not_found else - Gitlab::SlashCommands::Presenters::Deploy.new(actions).too_many_actions + deployment = action.play(current_user) + + Gitlab::SlashCommands::Presenters::Deploy + .new(deployment).present(from, to) end end private - def play!(from, to, action) - action.play(current_user) - end - - def find_actions(from, to) + def find_action(from, to) environment = project.environments.find_by(name: from) - return [] unless environment + return unless environment - environment.actions_for(to).select(&:starts_environment?) + actions = environment.actions_for(to).select do |action| + action.starts_environment? + end + + if actions.many? + actions.find { |action| action.name == to.to_s } + else + actions.first + end end end end diff --git a/lib/gitlab/slash_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb index 87ea19b8806..3d96982b820 100644 --- a/lib/gitlab/slash_commands/issue_command.rb +++ b/lib/gitlab/slash_commands/issue_command.rb @@ -2,7 +2,7 @@ module Gitlab module SlashCommands class IssueCommand < BaseCommand def self.available?(project) - project.issues_enabled? && project.default_issues_tracker? + project.issues_enabled? end def collection diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 27696436574..e13808a2720 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -2,7 +2,7 @@ module Gitlab module SlashCommands module Presenters class Base - include Gitlab::Routing.url_helpers + include Gitlab::Routing def initialize(resource = nil) @resource = resource diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb index b8dc77bd37b..ebae0f57f9b 100644 --- a/lib/gitlab/slash_commands/presenters/deploy.rb +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -3,17 +3,14 @@ module Gitlab module Presenters class Deploy < Presenters::Base def present(from, to) - message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + message = "Deployment started from #{from} to #{to}. " \ + "[Follow its progress](#{resource_url})." in_channel_response(text: message) end - def no_actions - ephemeral_response(text: "No action found to be executed") - end - - def too_many_actions - ephemeral_response(text: "Too many actions defined") + def action_not_found + ephemeral_response(text: "Couldn't find a deployment manual action.") end end end diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb new file mode 100644 index 00000000000..5e89e12b2b1 --- /dev/null +++ b/lib/gitlab/sql/glob.rb @@ -0,0 +1,22 @@ +module Gitlab + module SQL + module Glob + extend self + + # Convert a simple glob pattern with wildcard (*) to SQL LIKE pattern + # with SQL expression + def to_like(pattern) + <<~SQL + REPLACE(REPLACE(REPLACE(#{pattern}, + #{q('%')}, #{q('\\%')}), + #{q('_')}, #{q('\\_')}), + #{q('*')}, #{q('%')}) + SQL + end + + def q(string) + ActiveRecord::Base.connection.quote(string) + end + end + end +end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb new file mode 100644 index 00000000000..7ce2e9d636e --- /dev/null +++ b/lib/gitlab/untrusted_regexp.rb @@ -0,0 +1,49 @@ +module Gitlab + # An untrusted regular expression is any regexp containing patterns sourced + # from user input. + # + # Ruby's built-in regular expression library allows patterns which complete in + # exponential time, permitting denial-of-service attacks. + # + # Not all regular expression features are available in untrusted regexes, and + # there is a strict limit on total execution time. See the RE2 documentation + # at https://github.com/google/re2/wiki/Syntax for more details. + class UntrustedRegexp + delegate :===, to: :regexp + + def initialize(pattern) + @regexp = RE2::Regexp.new(pattern, log_errors: false) + + raise RegexpError.new(regexp.error) unless regexp.ok? + end + + def replace_all(text, rewrite) + RE2.GlobalReplace(text, regexp, rewrite) + end + + def scan(text) + matches = scan_regexp.scan(text).to_a + matches.map!(&:first) if regexp.number_of_capturing_groups.zero? + matches + end + + def replace(text, rewrite) + RE2.Replace(text, regexp, rewrite) + end + + private + + attr_reader :regexp + + # RE2 scan operates differently to Ruby scan when there are no capture + # groups, so work around it + def scan_regexp + @scan_regexp ||= + if regexp.number_of_capturing_groups.zero? + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 23af9318d1a..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Gitlab::Routing.url_helpers + include Gitlab::Routing include GitlabRoutingHelper include ActionView::RecordIdentifier @@ -23,9 +23,9 @@ module Gitlab when WikiPage wiki_page_url when ProjectSnippet - project_snippet_url(object) + project_snippet_url(object.project, object) when Snippet - personal_snippet_url(object) + snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end @@ -52,26 +52,24 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) + snippet = object.noteable if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) else - project_snippet_url(snippet, anchor: dom_id(object)) + project_snippet_url(snippet.project, snippet, anchor: dom_id(object)) end end end def wiki_page_url - namespace_project_wiki_url(object.wiki.project.namespace, object.wiki.project, object.slug) + project_wiki_url(object.wiki.project, object.slug) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index bcba2e3e1b6..e0ac21305a5 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -20,13 +20,15 @@ module Gitlab counts: { boards: Board.count, ci_builds: ::Ci::Build.count, - ci_pipelines: ::Ci::Pipeline.count, + ci_internal_pipelines: ::Ci::Pipeline.internal.count, + ci_external_pipelines: ::Ci::Pipeline.external.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, ci_pipeline_schedules: ::Ci::PipelineSchedule.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: Environment.count, + in_review_folder: Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, keys: Key.count, @@ -37,14 +39,14 @@ module Gitlab notes: Note.count, pages_domains: PagesDomain.count, projects: Project.count, - projects_prometheus_active: PrometheusService.active.count, + projects_imported_from_github: Project.where(import_type: 'github').count, protected_branches: ProtectedBranch.count, releases: Release.count, snippets: Snippet.count, todos: Todo.count, uploads: Upload.count, web_hooks: WebHook.count - } + }.merge(services_usage) } end @@ -61,6 +63,18 @@ module Gitlab usage_data end + + def services_usage + types = { + JiraService: :projects_jira_active, + SlackService: :projects_slack_notifications_active, + SlackSlashCommandsService: :projects_slack_slash_active, + PrometheusService: :projects_prometheus_active + } + + results = Service.unscoped.where(type: types.keys, active: true).group(:type).count + results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + end end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 3b922da7ced..d9a5af09f08 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,5 +1,11 @@ module Gitlab class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + attr_reader :user, :project def initialize(user, project: nil) @@ -28,43 +34,47 @@ module Gitlab true end - def can_create_tag?(ref) + request_cache def can_create_tag?(ref) return false unless can_access_git? - if ProtectedTag.protected?(project, ref) - project.protected_tags.protected_ref_accessible_to?(ref, user, action: :create) + if protected?(ProtectedTag, project, ref) + protected_tag_accessible_to?(ref, action: :create) else user.can?(:push_code, project) end end - def can_delete_branch?(ref) + request_cache def can_delete_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) user.can?(:delete_protected_branch, project) else user.can?(:push_code, project) end end - def can_push_to_branch?(ref) + def can_update_branch?(ref) + can_push_to_branch?(ref) || can_merge_to_branch?(ref) + end + + request_cache def can_push_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) + if protected?(ProtectedBranch, project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) + protected_branch_accessible_to?(ref, action: :push) else user.can?(:push_code, project) end end - def can_merge_to_branch?(ref) + request_cache def can_merge_to_branch?(ref) return false unless can_access_git? - if ProtectedBranch.protected?(project, ref) - project.protected_branches.protected_ref_accessible_to?(ref, user, action: :merge) + if protected?(ProtectedBranch, project, ref) + protected_branch_accessible_to?(ref, action: :merge) else user.can?(:push_code, project) end @@ -81,5 +91,23 @@ module Gitlab def can_access_git? user && user.can?(:access_git) end + + def protected_branch_accessible_to?(ref, action:) + ProtectedBranch.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_branches) + end + + def protected_tag_accessible_to?(ref, action:) + ProtectedTag.protected_ref_accessible_to?( + ref, user, + action: action, + protected_refs: project.protected_tags) + end + + request_cache def protected?(kind, project, ref) + kind.protected?(project, ref) + end end end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index dbfe0941e4d..841fb681435 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -15,6 +15,11 @@ module Gitlab super(user, action, overriden_subject || subject) end + # delegate all #can? queries to the subject + def declarative_policy_delegate + subject + end + class_methods do def presenter? true diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 36e5b5041a6..c60bd91ea6e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -28,7 +28,7 @@ module Gitlab def levels_for_user(user = nil) return [PUBLIC] unless user - if user.admin? + if user.full_private_access? [PRIVATE, INTERNAL, PUBLIC] elsif user.external? [PUBLIC] @@ -89,12 +89,12 @@ module Gitlab end def level_name(level) - level_name = 'Unknown' + level_name = N_('VisibilityLevel|Unknown') options.each do |name, lvl| level_name = name if lvl == level.to_i end - level_name + s_(level_name) end def level_value(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f96ee69096d..3f25e463412 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -25,27 +25,28 @@ module Gitlab RepoPath: repo_path } - if Gitlab.config.gitaly.enabled - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - - feature_enabled = case action.to_s - when 'git_receive_pack' - Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) - when 'git_upload_pack' - Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) - when 'info_refs' - true - else - raise "Unsupported action: #{action}" - end - if feature_enabled - params[:GitalyAddress] = server[:address] # This field will be deprecated - params[:GitalyServer] = server - end + server = { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } + params[:Repository] = repository.gitaly_repository.to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?( + :post_upload_pack, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT + ) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + if feature_enabled + params[:GitalyAddress] = server[:address] # This field will be deprecated + params[:GitalyServer] = server end params @@ -64,10 +65,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -178,7 +190,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -194,6 +206,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end |