diff options
Diffstat (limited to 'lib')
88 files changed, 1219 insertions, 679 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f4796f311a5..09a88869063 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -314,12 +314,35 @@ module API expose :id end + class MergeRequestSimple < ProjectEntity + expose :title + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end + end + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].upvotes + else + merge_request.upvotes + end + end + expose :downvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].downvotes + else + merge_request.downvotes + end + end expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :label_names, as: :labels + expose :labels do |merge_request, options| + # Avoids an N+1 query since labels are preloaded + merge_request.labels.map(&:title).sort + end expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds @@ -598,7 +621,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ef2c08e902c..8b007869dc3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -101,7 +101,7 @@ module API end get "/broadcast_message" do - if message = BroadcastMessage.current.last + if message = BroadcastMessage.current&.last present message, with: Entities::BroadcastMessage else {} @@ -150,7 +150,7 @@ module API # # begin # repository = wiki? ? project.wiki.repository : project.repository - # Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive + # Gitlab::GitalyClient::NotificationService.new(repository.raw_repository).post_receive # rescue GRPC::Unavailable => e # render_api_error!(e, 500) # end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index d419d345ec5..ac33b2b801c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -10,6 +10,8 @@ module API resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints + helpers ::Gitlab::IssuableMetadata + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -42,10 +44,14 @@ module API args[:label_name] = args.delete(:labels) merge_requests = MergeRequestsFinder.new(current_user, args).execute - .inc_notes_with_associations - .preload(:target_project, :author, :assignee, :milestone, :merge_request_diff) + .reorder(args[:order_by] => args[:sort]) + merge_requests = paginate(merge_requests) + .preload(:target_project) + + return merge_requests if args[:view] == 'simple' - merge_requests.reorder(args[:order_by] => args[:sort]) + merge_requests + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) end params :optional_params_ce do @@ -76,6 +82,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' use :pagination end get ":id/merge_requests" do @@ -83,7 +90,17 @@ module API merge_requests = find_merge_requests(project_id: user_project.id) - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + options = { with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options end desc 'Create a merge request' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d598f9a62a2..b19095d1252 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -65,6 +65,7 @@ module API :shared_runners_enabled, :sidekiq_throttling_enabled, :sign_in_text, + :password_authentication_enabled, :signin_enabled, :signup_enabled, :terminal_max_session_time, @@ -95,7 +96,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -176,6 +179,10 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else diff --git a/lib/api/users.rb b/lib/api/users.rb index c469751c31c..81c68ea2658 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -421,7 +421,16 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic + entity = + if sudo? + Entities::UserWithPrivateDetails + elsif current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end + + present current_user, with: entity end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c848f52723b..3759250f7f6 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -161,7 +161,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 748d6b97d4f..202011cfcbe 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,7 +44,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -116,7 +118,7 @@ module API :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, :user_oauth_applications, :user_default_external, :signup_enabled, :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication, :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, @@ -126,7 +128,13 @@ module API :housekeeping_enabled, :terminal_max_session_time end put "application/settings" do - if current_settings.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 56ad2c77c7d..cf3a0336792 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -80,6 +80,8 @@ module Ci artifacts: job[:artifacts], cache: job[:cache], dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], after_script: job[:after_script], environment: job[:environment] }.compact } diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index d9959bc1aff..b1eb1a6cef1 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -8,7 +8,12 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' +require 'thread' + module DeclarativePolicy + CLASS_CACHE_MUTEX = Mutex.new + CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE + class << self def policy_for(user, subject, opts = {}) cache = opts[:cache] || {} @@ -23,7 +28,36 @@ module DeclarativePolicy subject = find_delegate(subject) - subject.class.ancestors.each do |klass| + class_for_class(subject.class) + end + + private + + # This method is heavily cached because there are a lot of anonymous + # modules in play in a typical rails app, and #name performs quite + # slowly for anonymous classes and modules. + # + # See https://bugs.ruby-lang.org/issues/11119 + # + # if the above bug is resolved, this caching could likely be removed. + def class_for_class(subject_class) + unless subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + CLASS_CACHE_MUTEX.synchronize do + # re-check in case of a race + break if subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + + policy_class = compute_class_for_class(subject_class) + subject_class.instance_variable_set(CLASS_CACHE_IVAR, policy_class) + end + end + + policy_class = subject_class.instance_variable_get(CLASS_CACHE_IVAR) + raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class + end + + def compute_class_for_class(subject_class) + subject_class.ancestors.each do |klass| next unless klass.name begin @@ -37,12 +71,8 @@ module DeclarativePolicy nil end end - - raise "no policy for #{subject.class.name}" end - private - def find_delegate(subject) seen = Set.new diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index b8cc60074c7..0804edba016 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -21,11 +21,14 @@ module DeclarativePolicy private def id_for(obj) - if obj.respond_to?(:id) && obj.id - obj.id.to_s - else - "##{obj.object_id}" - end + id = + begin + obj.id + rescue NoMethodError + nil + end + + id || "##{obj.object_id}" end end end diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb index 9d7cf6b9726..51c4a8b2bbe 100644 --- a/lib/declarative_policy/condition.rb +++ b/lib/declarative_policy/condition.rb @@ -82,13 +82,14 @@ module DeclarativePolicy # depending on the scope, we may cache only by the user or only by # the subject, resulting in sharing across different policy objects. def cache_key - case @condition.scope - when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" - when :user then "/dp/condition/#{@condition.key}/#{user_key}" - when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" - when :global then "/dp/condition/#{@condition.key}" - else raise 'invalid scope' - end + @cache_key ||= + case @condition.scope + when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" + when :user then "/dp/condition/#{@condition.key}/#{user_key}" + when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" + when :global then "/dp/condition/#{@condition.key}" + else raise 'invalid scope' + end end def user_key diff --git a/lib/feature.rb b/lib/feature.rb index 363f66ba60e..4bd29aed687 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -57,5 +57,11 @@ class Feature Flipper.new(adapter) end end + + # This method is called from config/initializers/flipper.rb and can be used + # to register Flipper groups. + # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups + def register_feature_groups + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ccb5d886bab..9bed81e7327 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,7 +37,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -48,6 +48,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d95ecd7b291..b0741b1fba7 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,24 +1,45 @@ module Gitlab module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue - .new(BackgroundMigrationWorker.sidekiq_options['queue']) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - queue.each do |job| - migration_class, migration_args = job.args + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args - next unless migration_class == steal_class + next unless job.queue == self.queue + next unless migration_class == steal_class - perform(migration_class, migration_args) + begin + perform(migration_class, migration_args, retries: 3) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) - job.delete + raise + end + end end end + ## + # Performs a background migration. + # # class_name - The name of the background migration class as defined in the # Gitlab::BackgroundMigration namespace. # diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb new file mode 100644 index 00000000000..0881244ed49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -0,0 +1,26 @@ +module Gitlab + module BackgroundMigration + class MigrateSystemUploadsToNewFolder + include Gitlab::Database::MigrationHelpers + attr_reader :old_folder, :new_folder + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + include EachBatch + end + + def perform(old_folder, new_folder) + replace_sql = replace_sql(uploads[:path], old_folder, new_folder) + affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) + + affected_uploads.each_batch do |batch| + batch.update_all("path = #{replace_sql}") + end + end + + def uploads + Arel::Table.new('uploads') + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb new file mode 100644 index 00000000000..f1a04affd38 --- /dev/null +++ b/lib/gitlab/cache/request_cache.rb @@ -0,0 +1,94 @@ +module Gitlab + module Cache + # This module provides a simple way to cache values in RequestStore, + # and the cache key would be based on the class name, method name, + # optionally customized instance level values, optionally customized + # method level values, and optional method arguments. + # + # A simple example: + # + # class UserAccess + # extend Gitlab::Cache::RequestCache + # + # request_cache_key do + # [user&.id, project&.id] + # end + # + # request_cache def can_push_to_branch?(ref) + # # ... + # end + # end + # + # This way, the result of `can_push_to_branch?` would be cached in + # `RequestStore.store` based on the cache key. If RequestStore is not + # currently active, then it would be stored in a hash saved in an + # instance variable, so the cache logic would be the same. + # Here's another example using customized method level values: + # + # class Commit + # extend Gitlab::Cache::RequestCache + # + # def author + # User.find_by_any_email(author_email.downcase) + # end + # request_cache(:author) { author_email.downcase } + # end + # + # So that we could have different strategies for different methods + # + module RequestCache + def self.extended(klass) + return if klass < self + + extension = Module.new + klass.const_set(:RequestCacheExtension, extension) + klass.prepend(extension) + end + + def request_cache_key(&block) + if block_given? + @request_cache_key = block + else + @request_cache_key + end + end + + def request_cache(method_name, &method_key_block) + const_get(:RequestCacheExtension).module_eval do + cache_key_method_name = "#{method_name}_cache_key" + + define_method(method_name) do |*args| + store = + if RequestStore.active? + RequestStore.store + else + ivar_name = # ! and ? cannot be used as ivar name + "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" + + instance_variable_get(ivar_name) || + instance_variable_set(ivar_name, {}) + end + + key = __send__(cache_key_method_name, args) + + store.fetch(key) { store[key] = super(*args) } + end + + define_method(cache_key_method_name) do |args| + klass = self.class + + instance_key = instance_exec(&klass.request_cache_key) if + klass.request_cache_key + + method_key = instance_exec(&method_key_block) if method_key_block + + [klass.name, method_name, *instance_key, *method_key, *args] + .join(':') + end + + private cache_key_method_name + end + end + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c4c0623df6c..5d6977106d6 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -69,12 +69,12 @@ module Gitlab return unless valid? return unless regex - regex = Regexp.new(regex) + regex = Gitlab::UntrustedRegexp.new(regex) match = "" reverse_line do |line| - matches = line.scan(regex) + matches = regex.scan(line) next unless matches.is_a?(Array) next if matches.empty? diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 818b3d9c46b..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available end end @@ -33,12 +33,7 @@ module Gitlab def uncached_application_settings return fake_application_settings unless connect_to_db? - # This loads from the database into the cache, so handle Redis errors - begin - db_settings = ::ApplicationSetting.current - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - end + db_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0643c56db9b..69ca9aa596b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -140,6 +140,8 @@ module Gitlab return add_foreign_key(source, target, column: column, on_delete: on_delete) + else + on_delete = 'SET NULL' if on_delete == :nullify end disable_statement_timeout @@ -155,7 +157,7 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete}" : ''} + #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} NOT VALID; EOF diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a0f46594eb1..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -26,17 +26,17 @@ module Gitlab EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -45,24 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..0deaab01b5b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC for #load_blame. + module Gitlab module Git class Blame @@ -24,6 +26,7 @@ module Gitlab private + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) # Read in binary mode to ensure ASCII-8BIT diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ea386c2ddcb..db6cfc9671f 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -27,7 +29,7 @@ module Gitlab path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) return unless entry case entry.type @@ -85,10 +87,10 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else blob = repository.lookup(sha) - + new( id: blob.oid, size: blob.size, @@ -107,6 +109,8 @@ module Gitlab detect && detect[:type] == :binary end + private + # Recursive search of blob id by path # # Ex. @@ -178,7 +182,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 9c0606d780a..76a562f356e 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -97,7 +98,15 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.commits_between(base, head).map do |commit| + commits = Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.between(base, head) + else + repo.commits_between(base, head) + end + end + + commits.map do |commit| decorate(commit) end rescue Rugged::ReferenceError @@ -124,6 +133,7 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) actual_options = options.dup @@ -208,6 +218,8 @@ module Gitlab init_from_hash(raw_commit) elsif raw_commit.is_a?(Rugged::Commit) init_from_rugged(raw_commit) + elsif raw_commit.is_a?(Gitaly::GitCommit) + init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end @@ -243,6 +255,8 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff diff_from_parent.patch end @@ -367,6 +381,22 @@ module Gitlab @parent_ids = commit.parents.map(&:oid) end + def init_from_gitaly(commit) + @raw_commit = commit + @id = commit.id + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + @message = (commit.body.presence || commit.subject).dup + @authored_date = Time.at(commit.author.date.seconds) + @author_name = commit.author.name.dup + @author_email = commit.author.email.dup + @committed_date = Time.at(commit.committer.date.seconds) + @committer_name = commit.committer.name.dup + @committer_email = commit.committer.email.dup + @parent_ids = commit.parent_ids + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 78e440395a5..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index cf7829a583b..cf95f673667 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -81,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 555894907cc..0d8fe185ac5 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index f4e3b5e5129..4a43b9b444d 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class GitmodulesParser diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 5042916343b..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 666743006e5..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e51966313d4..63eebadff2e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -80,16 +80,10 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389 + def branches(sort_by: nil) + branches_filter(sort_by: sort_by) end def reload_rugged @@ -107,7 +101,10 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) @@ -115,7 +112,7 @@ module Gitlab if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -162,6 +159,8 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags rugged.references.each("refs/tags/*").map do |ref| message = nil @@ -174,7 +173,8 @@ module Gitlab end end - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) end.sort_by(&:name) end @@ -204,13 +204,6 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end @@ -297,28 +290,6 @@ module Gitlab (size.to_f / 1024).round(2) end - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) - end - - greps - end - # Use the Rugged Walker API to build an array of commits. # # Usage. @@ -331,85 +302,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -454,7 +350,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -553,6 +449,7 @@ module Gitlab # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| if is_enabled @@ -567,6 +464,8 @@ module Gitlab end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) gitaly_migrate(:commit_count) do |is_enabled| if is_enabled @@ -779,7 +678,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -838,6 +738,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -864,6 +765,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -905,8 +807,113 @@ module Gitlab Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) + end + private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 @@ -930,7 +937,7 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT - gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Blob::MAX_DATA_DISPLAY_SIZE) + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) found_module = GitmodulesParser.new(gitmodules.data).parse[path] found_module && found_module['url'] @@ -1078,73 +1085,6 @@ module Gitlab index end - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1173,14 +1113,6 @@ module Gitlab end end - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) - end - - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) - end - def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound => e diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b6d4e6cfe46..8122ff0e81f 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,6 +12,8 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' @@ -40,6 +44,8 @@ module Gitlab end end + private + # Recursive search of tree id for path # # Ex. diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob_service.rb index 0c398b46a08..7ea8e8d0857 100644 --- a/lib/gitlab/gitaly_client/blob.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,10 +1,10 @@ module Gitlab module GitalyClient - class Blob + class BlobService def initialize(repository) @gitaly_repo = repository.gitaly_repository end - + def get_blob(oid:, limit:) request = Gitaly::GetBlobRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit_service.rb index aafc0520664..8f5738fed06 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Commit + class CommitService # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -17,20 +17,20 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value end def diff_from_parent(commit, options = {}) request_params = commit_diff_request_params(commit, options) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) response.flat_map do |msg| msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end @@ -44,7 +44,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request) + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) entry = response.first return unless entry.oid.present? @@ -65,6 +65,17 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count end + def between(from, to) + request = Gitaly::CommitsBetweenRequest.new( + repository: @gitaly_repo, + from: from, + to: to + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + consume_commits_response(response) + end + private def commit_diff_request_params(commit, options = {}) @@ -77,6 +88,10 @@ module Gitlab paths: options.fetch(:paths, []) } end + + def consume_commits_response(response) + response.flat_map { |r| r.commits } + end end end end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notification_service.rb index 78ed433e6b8..326e6f7dafc 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Notifications + class NotificationService # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository @@ -10,7 +10,7 @@ module Gitlab def post_receive GitalyClient.call( @storage, - :notifications, + :notification_service, :post_receive, Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) ) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref_service.rb index 6edc69de078..2c3d53410ac 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Ref + class RefService include Gitlab::EncodingHelper # 'repository' is a Gitlab::Git::Repository @@ -12,19 +12,19 @@ module Gitlab def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -34,7 +34,7 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup) + encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) end def count_tag_names @@ -48,7 +48,7 @@ module Gitlab def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref, :find_local_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) consume_branches_response(response) end @@ -72,11 +72,39 @@ module Gitlab Gitlab::Git::Branch.new( @repository, encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) ) end end end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name.dup, + author_email: response.commit_author.email.dup, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name.dup, + committer_email: response.commit_committer.email.dup + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 70da4080cae..bebde857b16 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -35,9 +35,9 @@ module Gitlab repository_storages.flat_map do |storage_name| tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name), + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name), + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name) ].flatten end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..3dcb28a193c 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -20,7 +20,7 @@ module Gitlab [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) + metric("#{metric_prefix}_latency_seconds", elapsed) ] end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f3d489aad0d..a1b896c9511 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -12,8 +12,11 @@ module Gitlab 'zh_HK' => '繁體中文(香港)', 'zh_TW' => '繁體中文(臺灣)', 'bg' => 'български', + 'ru' => 'Русский', 'eo' => 'Esperanto', - 'it' => 'Italiano' + 'it' => 'Italiano', + 'uk' => 'Українська', + 'ja' => '日本語' }.freeze def available_locales diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb deleted file mode 100644 index b3da360be8f..00000000000 --- a/lib/gitlab/metrics/connection_rack_middleware.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module Metrics - class ConnectionRackMiddleware - def initialize(app) - @app = app - end - - def self.rack_request_count - @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count') - end - - def self.rack_response_count - @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count') - end - - def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count') - end - - def self.rack_execution_time - @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10]) - end - - def call(env) - method = env['REQUEST_METHOD'].downcase - started = Time.now.to_f - begin - ConnectionRackMiddleware.rack_request_count.increment(method: method) - - status, headers, body = @app.call(env) - - ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status) - [status, headers, body] - rescue - ConnectionRackMiddleware.rack_uncaught_errors_count.increment - raise - ensure - elapsed = Time.now.to_f - started - ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed) - end - end - end - end -end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index fb7bbc7cfc7..460dab47276 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -6,9 +6,11 @@ module Gitlab include Gitlab::CurrentSettings def metrics_folder_present? - ENV.has_key?('prometheus_multiproc_dir') && - ::Dir.exist?(ENV['prometheus_multiproc_dir']) && - ::File.writable?(ENV['prometheus_multiproc_dir']) + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) end def prometheus_metrics_enabled? diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b3f453e506d..3f2bbd9f6a6 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -101,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index d81f825ef96..60a32d5d5ea 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -49,7 +49,6 @@ module Gitlab sent_notifications services snippets - system teams u unicorn_test diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 2da2ce45ebc..56112ec2301 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -2,7 +2,8 @@ module Gitlab module PerformanceBar include Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids'.freeze + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes def self.enabled?(user = nil) return false unless user && allowed_group_id @@ -15,7 +16,7 @@ module Gitlab end def self.allowed_user_ids - Rails.cache.fetch(ALLOWED_USER_IDS_KEY) do + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) if group diff --git a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb deleted file mode 100644 index d939a6ea18d..00000000000 --- a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This solves a bug with a X-Senfile header that wouldn't be set properly, see -# https://github.com/peek/peek-performance_bar/pull/27 -module Gitlab - module PerformanceBar - module PeekPerformanceBarWithRackBody - def call(env) - @env = env - reset_stats - - @total_requests += 1 - first_request if @total_requests == 1 - - env['process.request_start'] = @start.to_f - env['process.total_requests'] = total_requests - - status, headers, body = @app.call(env) - body = Rack::BodyProxy.new(body) { record_request } - [status, headers, body] - end - end - end -end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 574ae8731a5..67fee8c227d 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -1,4 +1,5 @@ # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb +# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb module Gitlab module PerformanceBar module PeekQueryTracker @@ -23,14 +24,20 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if RequestStore.active? && RequestStore.store[:peek_enabled] - track_query(data[:sql].strip, data[:binds], start, finish) + # data[:cached] is only available starting from Rails 5.1.0 + # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 + # Before that, data[:name] was set to 'CACHE' + # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end end end end def track_query(raw_query, bindings, start, finish) query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: '%.3f' % query.duration, sql: query.formatted_query } + query_info = { duration: query.duration.round(3), sql: query.formatted_query } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 877aa6e6a28..f3952657983 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -18,7 +18,11 @@ module Gitlab mapping = @map.find { |mapping| mapping[:source] === path } return unless mapping - path.sub(mapping[:source], mapping[:public]) + if mapping[:source].is_a?(String) + path.sub(mapping[:source], mapping[:public]) + else + mapping[:source].replace(path, mapping[:public]) + end end private @@ -35,7 +39,7 @@ module Gitlab source_pattern = source_pattern[1...-1].gsub('\/', '/') begin - source_pattern = /\A#{source_pattern}\z/ + source_pattern = Gitlab::UntrustedRegexp.new('\A' + source_pattern + '\z') rescue RegexpError => e raise FormatError, "Route map entry source is not a valid regular expression: #{e}" end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0baea092e6a..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,3 +1,6 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab @@ -68,6 +71,7 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'add-project', storage, "#{name}.git"]) @@ -81,6 +85,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. @@ -99,6 +104,7 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced @@ -115,6 +121,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', storage, "#{path}.git", "#{new_path}.git"]) @@ -129,6 +136,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', forked_from_storage, "#{path}.git", forked_to_storage, @@ -143,6 +151,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'rm-project', storage, "#{name}.git"]) @@ -194,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -207,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -216,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -241,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb new file mode 100644 index 00000000000..8b43f0053d6 --- /dev/null +++ b/lib/gitlab/untrusted_regexp.rb @@ -0,0 +1,53 @@ +module Gitlab + # An untrusted regular expression is any regexp containing patterns sourced + # from user input. + # + # Ruby's built-in regular expression library allows patterns which complete in + # exponential time, permitting denial-of-service attacks. + # + # Not all regular expression features are available in untrusted regexes, and + # there is a strict limit on total execution time. See the RE2 documentation + # at https://github.com/google/re2/wiki/Syntax for more details. + class UntrustedRegexp + delegate :===, to: :regexp + + def initialize(pattern) + @regexp = RE2::Regexp.new(pattern, log_errors: false) + + raise RegexpError.new(regexp.error) unless regexp.ok? + end + + def replace_all(text, rewrite) + RE2.GlobalReplace(text, regexp, rewrite) + end + + def scan(text) + scan_regexp.scan(text).map do |match| + if regexp.number_of_capturing_groups == 0 + match.first + else + match + end + end + end + + def replace(text, rewrite) + RE2.Replace(text, regexp, rewrite) + end + + private + + attr_reader :regexp + + # RE2 scan operates differently to Ruby scan when there are no capture + # groups, so work around it + def scan_regexp + @scan_regexp ||= + if regexp.number_of_capturing_groups == 0 + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 35792d2d67f..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -52,15 +52,13 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) + snippet = object.noteable if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f19b325a126..dba071d7e47 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -39,6 +39,7 @@ module Gitlab notes: Note.count, pages_domains: PagesDomain.count, projects: Project.count, + projects_imported_from_github: Project.where(import_type: 'github').count, projects_prometheus_active: PrometheusService.active.count, protected_branches: ProtectedBranch.count, releases: Release.count, diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 3b922da7ced..8e91ee7287c 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,5 +1,11 @@ module Gitlab class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + attr_reader :user, :project def initialize(user, project: nil) @@ -28,7 +34,7 @@ module Gitlab true end - def can_create_tag?(ref) + request_cache def can_create_tag?(ref) return false unless can_access_git? if ProtectedTag.protected?(project, ref) @@ -38,7 +44,7 @@ module Gitlab end end - def can_delete_branch?(ref) + request_cache def can_delete_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) @@ -48,7 +54,7 @@ module Gitlab end end - def can_push_to_branch?(ref) + request_cache def can_push_to_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) @@ -60,7 +66,7 @@ module Gitlab end end - def can_merge_to_branch?(ref) + request_cache def can_merge_to_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 48f3d950779..c60bd91ea6e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -89,12 +89,12 @@ module Gitlab end def level_name(level) - level_name = 'Unknown' + level_name = N_('VisibilityLevel|Unknown') options.each do |name, lvl| level_name = name if lvl == level.to_i end - level_name + s_(level_name) end def level_value(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 4aef23b6aee..916ef365d78 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -62,10 +62,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:project_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -176,7 +187,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -192,6 +203,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d6..564aa141952 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -5,12 +5,12 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| cursor = REDIS_SCAN_START_STOP loop do cursor, keys = redis.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: REDIS_CLEAR_BATCH_SIZE ) diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b27f7475115..b48e4dce445 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -5,7 +5,7 @@ namespace :gettext do # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate folders = %W(app lib config #{locale_path}).join(',') - exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( "{#{folders}}/**/*.{#{exts}}" |