diff options
| author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-07-20 11:48:23 +0200 |
|---|---|---|
| committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-07-20 11:48:23 +0200 |
| commit | 54efa041d70707eb28326bb23220ebe8d6efb8aa (patch) | |
| tree | 63b1bc838861f2393b5bc283778299e4cf96a70f /lib | |
| parent | bb67b4749b5b4c62d4235c90dc0320967f850cdd (diff) | |
| parent | 445cd22c72ca6fbfdcf18d67fa859c4b5b9e2a6c (diff) | |
| download | gitlab-ce-54efa041d70707eb28326bb23220ebe8d6efb8aa.tar.gz | |
Merge branch 'master' into backstage/gb/migrate-stages-statuses
* master: (319 commits)
remove redundant changelog entries
Merge branch '24570-use-re2-for-user-supplied-regexp-9-3' into 'security-9-3'
Merge branch '33303-404-for-unauthorized-project' into 'security-9-3'
Merge branch '33359-pers-snippet-files-location' into 'security-9-3'
Merge branch 'bvl-remove-appearance-symlink' into 'security-9-3'
Hide description about protected branches to non-member
Update CHANGELOG.md for 9.0.11
Update CHANGELOG.md for 9.1.8
Update CHANGELOG.md for 8.17.7
Update CHANGELOG.md for 9.2.8
Update CHANGELOG.md for 9.3.8
Respect blockquote line breaks in markdown
35209 Add wip message to new navigation preference section
Add github imported projects count to usage data
Add versions to Prometheus metrics doc
Add Bulgarian translations of Pipeline Schedules
Add Esperanto translations of Pipeline Schedules
Add Traditional Chinese in HongKong translations of Pipeline Schedules
Add Simplified Chinese translations of Pipeline Schedules
Resolve "Clarify k8s service keys"
...
Conflicts:
db/schema.rb
Diffstat (limited to 'lib')
88 files changed, 1219 insertions, 679 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f4796f311a5..09a88869063 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -314,12 +314,35 @@ module API expose :id end + class MergeRequestSimple < ProjectEntity + expose :title + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end + end + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch - expose :upvotes, :downvotes + expose :upvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].upvotes + else + merge_request.upvotes + end + end + expose :downvotes do |merge_request, options| + if options[:issuable_metadata] + options[:issuable_metadata][merge_request.id].downvotes + else + merge_request.downvotes + end + end expose :author, :assignee, using: Entities::UserBasic expose :source_project_id, :target_project_id - expose :label_names, as: :labels + expose :labels do |merge_request, options| + # Avoids an N+1 query since labels are preloaded + merge_request.labels.map(&:title).sort + end expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds @@ -598,7 +621,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ef2c08e902c..8b007869dc3 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -101,7 +101,7 @@ module API end get "/broadcast_message" do - if message = BroadcastMessage.current.last + if message = BroadcastMessage.current&.last present message, with: Entities::BroadcastMessage else {} @@ -150,7 +150,7 @@ module API # # begin # repository = wiki? ? project.wiki.repository : project.repository - # Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive + # Gitlab::GitalyClient::NotificationService.new(repository.raw_repository).post_receive # rescue GRPC::Unavailable => e # render_api_error!(e, 500) # end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index d419d345ec5..ac33b2b801c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -10,6 +10,8 @@ module API resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints + helpers ::Gitlab::IssuableMetadata + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -42,10 +44,14 @@ module API args[:label_name] = args.delete(:labels) merge_requests = MergeRequestsFinder.new(current_user, args).execute - .inc_notes_with_associations - .preload(:target_project, :author, :assignee, :milestone, :merge_request_diff) + .reorder(args[:order_by] => args[:sort]) + merge_requests = paginate(merge_requests) + .preload(:target_project) + + return merge_requests if args[:view] == 'simple' - merge_requests.reorder(args[:order_by] => args[:sort]) + merge_requests + .preload(:notes, :author, :assignee, :milestone, :merge_request_diff, :labels) end params :optional_params_ce do @@ -76,6 +82,7 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' use :pagination end get ":id/merge_requests" do @@ -83,7 +90,17 @@ module API merge_requests = find_merge_requests(project_id: user_project.id) - present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + options = { with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project } + + if params[:view] == 'simple' + options[:with] = Entities::MergeRequestSimple + else + options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest') + end + + present merge_requests, options end desc 'Create a merge request' do diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d598f9a62a2..b19095d1252 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -65,6 +65,7 @@ module API :shared_runners_enabled, :sidekiq_throttling_enabled, :sign_in_text, + :password_authentication_enabled, :signin_enabled, :signup_enabled, :terminal_max_session_time, @@ -95,7 +96,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -176,6 +179,10 @@ module API put "application/settings" do attrs = declared_params(include_missing: false) + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else diff --git a/lib/api/users.rb b/lib/api/users.rb index c469751c31c..81c68ea2658 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -421,7 +421,16 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic + entity = + if sudo? + Entities::UserWithPrivateDetails + elsif current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end + + present current_user, with: entity end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c848f52723b..3759250f7f6 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -161,7 +161,8 @@ module API expose :id expose :default_projects_limit expose :signup_enabled - expose :signin_enabled + expose :password_authentication_enabled + expose :password_authentication_enabled, as: :signin_enabled expose :gravatar_enabled expose :sign_in_text expose :after_sign_up_text diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb index 748d6b97d4f..202011cfcbe 100644 --- a/lib/api/v3/settings.rb +++ b/lib/api/v3/settings.rb @@ -44,7 +44,9 @@ module API requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled' + mutually_exclusive :password_authentication_enabled, :signin_enabled optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' given require_two_factor_authentication: ->(val) { val } do requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' @@ -116,7 +118,7 @@ module API :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, :user_oauth_applications, :user_default_external, :signup_enabled, :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, - :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :after_sign_up_text, :password_authentication_enabled, :signin_enabled, :require_two_factor_authentication, :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, @@ -126,7 +128,13 @@ module API :housekeeping_enabled, :terminal_max_session_time end put "application/settings" do - if current_settings.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + + if attrs.has_key?(:signin_enabled) + attrs[:password_authentication_enabled] = attrs.delete(:signin_enabled) + end + + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 56ad2c77c7d..cf3a0336792 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -80,6 +80,8 @@ module Ci artifacts: job[:artifacts], cache: job[:cache], dependencies: job[:dependencies], + before_script: job[:before_script], + script: job[:script], after_script: job[:after_script], environment: job[:environment] }.compact } diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index d9959bc1aff..b1eb1a6cef1 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -8,7 +8,12 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' +require 'thread' + module DeclarativePolicy + CLASS_CACHE_MUTEX = Mutex.new + CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE + class << self def policy_for(user, subject, opts = {}) cache = opts[:cache] || {} @@ -23,7 +28,36 @@ module DeclarativePolicy subject = find_delegate(subject) - subject.class.ancestors.each do |klass| + class_for_class(subject.class) + end + + private + + # This method is heavily cached because there are a lot of anonymous + # modules in play in a typical rails app, and #name performs quite + # slowly for anonymous classes and modules. + # + # See https://bugs.ruby-lang.org/issues/11119 + # + # if the above bug is resolved, this caching could likely be removed. + def class_for_class(subject_class) + unless subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + CLASS_CACHE_MUTEX.synchronize do + # re-check in case of a race + break if subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) + + policy_class = compute_class_for_class(subject_class) + subject_class.instance_variable_set(CLASS_CACHE_IVAR, policy_class) + end + end + + policy_class = subject_class.instance_variable_get(CLASS_CACHE_IVAR) + raise "no policy for #{subject.class.name}" if policy_class.nil? + policy_class + end + + def compute_class_for_class(subject_class) + subject_class.ancestors.each do |klass| next unless klass.name begin @@ -37,12 +71,8 @@ module DeclarativePolicy nil end end - - raise "no policy for #{subject.class.name}" end - private - def find_delegate(subject) seen = Set.new diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index b8cc60074c7..0804edba016 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -21,11 +21,14 @@ module DeclarativePolicy private def id_for(obj) - if obj.respond_to?(:id) && obj.id - obj.id.to_s - else - "##{obj.object_id}" - end + id = + begin + obj.id + rescue NoMethodError + nil + end + + id || "##{obj.object_id}" end end end diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb index 9d7cf6b9726..51c4a8b2bbe 100644 --- a/lib/declarative_policy/condition.rb +++ b/lib/declarative_policy/condition.rb @@ -82,13 +82,14 @@ module DeclarativePolicy # depending on the scope, we may cache only by the user or only by # the subject, resulting in sharing across different policy objects. def cache_key - case @condition.scope - when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" - when :user then "/dp/condition/#{@condition.key}/#{user_key}" - when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" - when :global then "/dp/condition/#{@condition.key}" - else raise 'invalid scope' - end + @cache_key ||= + case @condition.scope + when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" + when :user then "/dp/condition/#{@condition.key}/#{user_key}" + when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" + when :global then "/dp/condition/#{@condition.key}" + else raise 'invalid scope' + end end def user_key diff --git a/lib/feature.rb b/lib/feature.rb index 363f66ba60e..4bd29aed687 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -57,5 +57,11 @@ class Feature Flipper.new(adapter) end end + + # This method is called from config/initializers/flipper.rb and can be used + # to register Flipper groups. + # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups + def register_feature_groups + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ccb5d886bab..9bed81e7327 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -37,7 +37,7 @@ module Gitlab rate_limit!(ip, success: result.success?, login: login) Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) - return result if result.success? || current_application_settings.signin_enabled? || Gitlab::LDAP::Config.enabled? + return result if result.success? || current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? # If sign-in is disabled and LDAP is not configured, recommend a # personal access token on failed auth attempts @@ -48,6 +48,10 @@ module Gitlab # Avoid resource intensive login checks if password is not provided return unless password.present? + # Nothing to do here if internal auth is disabled and LDAP is + # not configured + return unless current_application_settings.password_authentication_enabled? || Gitlab::LDAP::Config.enabled? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index bf2239ca150..baa1f802d8a 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -27,7 +27,7 @@ module Gitlab time = Time.now.utc.to_i key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d95ecd7b291..b0741b1fba7 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,24 +1,45 @@ module Gitlab module BackgroundMigration + def self.queue + @queue ||= BackgroundMigrationWorker.sidekiq_options['queue'] + end + # Begins stealing jobs from the background migrations queue, blocking the # caller until all jobs have been completed. # + # When a migration raises a StandardError is is going to be retries up to + # three times, for example, to recover from a deadlock. + # + # When Exception is being raised, it enqueues the migration again, and + # re-raises the exception. + # # steal_class - The name of the class for which to steal jobs. def self.steal(steal_class) - queue = Sidekiq::Queue - .new(BackgroundMigrationWorker.sidekiq_options['queue']) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - queue.each do |job| - migration_class, migration_args = job.args + [scheduled, enqueued].each do |queue| + queue.each do |job| + migration_class, migration_args = job.args - next unless migration_class == steal_class + next unless job.queue == self.queue + next unless migration_class == steal_class - perform(migration_class, migration_args) + begin + perform(migration_class, migration_args, retries: 3) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + BackgroundMigrationWorker # enqueue this migration again + .perform_async(migration_class, migration_args) - job.delete + raise + end + end end end + ## + # Performs a background migration. + # # class_name - The name of the background migration class as defined in the # Gitlab::BackgroundMigration namespace. # diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb new file mode 100644 index 00000000000..0881244ed49 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -0,0 +1,26 @@ +module Gitlab + module BackgroundMigration + class MigrateSystemUploadsToNewFolder + include Gitlab::Database::MigrationHelpers + attr_reader :old_folder, :new_folder + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + include EachBatch + end + + def perform(old_folder, new_folder) + replace_sql = replace_sql(uploads[:path], old_folder, new_folder) + affected_uploads = Upload.where(uploads[:path].matches("#{old_folder}%")) + + affected_uploads.each_batch do |batch| + batch.update_all("path = #{replace_sql}") + end + end + + def uploads + Arel::Table.new('uploads') + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9c2e09943b0..dba37892863 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -23,7 +23,7 @@ module Gitlab end def self.cached_results_for_projects(projects) - result = Gitlab::Redis.with do |redis| + result = Gitlab::Redis::Cache.with do |redis| redis.multi do projects.each do |project| cache_key = cache_key_for_project(project) @@ -100,19 +100,19 @@ module Gitlab end def load_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) end end def store_in_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.del(cache_key) end end @@ -120,7 +120,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| redis.exists(cache_key) end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb new file mode 100644 index 00000000000..f1a04affd38 --- /dev/null +++ b/lib/gitlab/cache/request_cache.rb @@ -0,0 +1,94 @@ +module Gitlab + module Cache + # This module provides a simple way to cache values in RequestStore, + # and the cache key would be based on the class name, method name, + # optionally customized instance level values, optionally customized + # method level values, and optional method arguments. + # + # A simple example: + # + # class UserAccess + # extend Gitlab::Cache::RequestCache + # + # request_cache_key do + # [user&.id, project&.id] + # end + # + # request_cache def can_push_to_branch?(ref) + # # ... + # end + # end + # + # This way, the result of `can_push_to_branch?` would be cached in + # `RequestStore.store` based on the cache key. If RequestStore is not + # currently active, then it would be stored in a hash saved in an + # instance variable, so the cache logic would be the same. + # Here's another example using customized method level values: + # + # class Commit + # extend Gitlab::Cache::RequestCache + # + # def author + # User.find_by_any_email(author_email.downcase) + # end + # request_cache(:author) { author_email.downcase } + # end + # + # So that we could have different strategies for different methods + # + module RequestCache + def self.extended(klass) + return if klass < self + + extension = Module.new + klass.const_set(:RequestCacheExtension, extension) + klass.prepend(extension) + end + + def request_cache_key(&block) + if block_given? + @request_cache_key = block + else + @request_cache_key + end + end + + def request_cache(method_name, &method_key_block) + const_get(:RequestCacheExtension).module_eval do + cache_key_method_name = "#{method_name}_cache_key" + + define_method(method_name) do |*args| + store = + if RequestStore.active? + RequestStore.store + else + ivar_name = # ! and ? cannot be used as ivar name + "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" + + instance_variable_get(ivar_name) || + instance_variable_set(ivar_name, {}) + end + + key = __send__(cache_key_method_name, args) + + store.fetch(key) { store[key] = super(*args) } + end + + define_method(cache_key_method_name) do |args| + klass = self.class + + instance_key = instance_exec(&klass.request_cache_key) if + klass.request_cache_key + + method_key = instance_exec(&method_key_block) if method_key_block + + [klass.name, method_name, *instance_key, *method_key, *args] + .join(':') + end + + private cache_key_method_name + end + end + end + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 1b081aa9b1d..e63e5437331 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -12,23 +12,23 @@ module Gitlab end def get - Gitlab::Redis.with do |redis| - data = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + data = redis.get(redis_shared_state_key) JSON.parse(data, symbolize_names: true) if data end end def store!(params) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| params = params.to_json - redis.set(redis_key, params, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME) token end end def delete - Gitlab::Redis.with do |redis| - redis.del(redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_shared_state_key) end end @@ -38,7 +38,7 @@ module Gitlab Devise.friendly_token(TOKEN_LENGTH) end - def redis_key + def redis_shared_state_key "gitlab:chat_names:#{token}" end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index ee034d9cc56..411f67f8ce7 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -12,7 +12,8 @@ module Gitlab class << self def from_commands(job) self.new(:script).tap do |step| - step.script = job.commands.split("\n") + step.script = job.options[:before_script].to_a + job.options[:script].to_a + step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index c4c0623df6c..5d6977106d6 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -69,12 +69,12 @@ module Gitlab return unless valid? return unless regex - regex = Regexp.new(regex) + regex = Gitlab::UntrustedRegexp.new(regex) match = "" reverse_line do |line| - matches = line.scan(regex) + matches = regex.scan(line) next unless matches.is_a?(Array) next if matches.empty? diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 818b3d9c46b..7fa02f3d7b3 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -25,7 +25,7 @@ module Gitlab def cached_application_settings begin ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL # In case Redis isn't running or the Redis UNIX socket file is not available end end @@ -33,12 +33,7 @@ module Gitlab def uncached_application_settings return fake_application_settings unless connect_to_db? - # This loads from the database into the cache, so handle Redis errors - begin - db_settings = ::ApplicationSetting.current - rescue ::Redis::BaseError, ::Errno::ENOENT - # In case Redis isn't running or the Redis UNIX socket file is not available - end + db_settings = ::ApplicationSetting.current # If there are pending migrations, it's possible there are columns that # need to be added to the application settings. To prevent Rake tasks diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0643c56db9b..69ca9aa596b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -140,6 +140,8 @@ module Gitlab return add_foreign_key(source, target, column: column, on_delete: on_delete) + else + on_delete = 'SET NULL' if on_delete == :nullify end disable_statement_timeout @@ -155,7 +157,7 @@ module Gitlab ADD CONSTRAINT #{key_name} FOREIGN KEY (#{column}) REFERENCES #{target} (id) - #{on_delete ? "ON DELETE #{on_delete}" : ''} + #{on_delete ? "ON DELETE #{on_delete.upcase}" : ''} NOT VALID; EOF diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 33f8939bc61..1a697396ff1 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -145,7 +145,7 @@ module Gitlab def track_rename(type, old_path, new_path) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.lpush(key, [old_path, new_path].to_json) redis.expire(key, 2.weeks.to_i) end @@ -155,7 +155,7 @@ module Gitlab def reverts_for_type(type) key = redis_key_for_type(type) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| failed_reverts = [] while rename_info = redis.lpop(key) diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 072fcfc65e6..21172ff8d93 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -2,17 +2,17 @@ module Gitlab module EtagCaching class Store EXPIRY_TIME = 20.minutes - REDIS_NAMESPACE = 'etag:'.freeze + SHARED_STATE_NAMESPACE = 'etag:'.freeze def get(key) - Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) } end def touch(key, only_if_missing: false) etag = generate_etag - Gitlab::Redis.with do |redis| - redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) end etag @@ -24,10 +24,10 @@ module Gitlab SecureRandom.hex end - def redis_key(key) + def redis_shared_state_key(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) - "#{REDIS_NAMESPACE}#{key}" + "#{SHARED_STATE_NAMESPACE}#{key}" end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index a0f46594eb1..3784f6c4947 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -26,17 +26,17 @@ module Gitlab EOS def self.cancel(key, uuid) - Gitlab::Redis.with do |redis| - redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + Gitlab::Redis::SharedState.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid]) end end - def self.redis_key(key) + def self.redis_shared_state_key(key) "gitlab:exclusive_lease:#{key}" end def initialize(key, timeout:) - @redis_key = self.class.redis_key(key) + @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @uuid = SecureRandom.uuid end @@ -45,24 +45,24 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis.with do |redis| - redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid + Gitlab::Redis::SharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis.with do |redis| - result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) + Gitlab::Redis::SharedState.with do |redis| + result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end end # Returns true if the key for this lease is set. def exists? - Gitlab::Redis.with do |redis| - redis.exists(@redis_key) + Gitlab::Redis::SharedState.with do |redis| + redis.exists(@redis_shared_state_key) end end end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb index 42140ecc993..2d20cd473a7 100644 --- a/lib/gitlab/git/attributes.rb +++ b/lib/gitlab/git/attributes.rb @@ -1,3 +1,8 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 66829a03c2e..0deaab01b5b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC for #load_blame. + module Gitlab module Git class Blame @@ -24,6 +26,7 @@ module Gitlab private + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/376 def load_blame cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) # Read in binary mode to ensure ASCII-8BIT diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ea386c2ddcb..db6cfc9671f 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: seems to be completely migrated (behind feature flags). + module Gitlab module Git class Blob @@ -27,7 +29,7 @@ module Gitlab path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::Commit.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) return unless entry case entry.type @@ -85,10 +87,10 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else blob = repository.lookup(sha) - + new( id: blob.oid, size: blob.size, @@ -107,6 +109,8 @@ module Gitlab detect && detect[:type] == :binary end + private + # Recursive search of blob id by path # # Ex. @@ -178,7 +182,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::Blob.new(repository).get_blob(oid: id, limit: -1).data + Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index d7975f88aaa..68116e775c6 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class BlobSnippet diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 124526e4b59..c53882787f1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,39 +1,10 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Branch < Ref - def initialize(repository, name, target) - if target.is_a?(Gitaly::FindLocalBranchResponse) - target = target_from_gitaly_local_branches_response(target) - end - - super(repository, name, target) - end - - def target_from_gitaly_local_branches_response(response) - # Git messages have no encoding enforcements. However, in the UI we only - # handle UTF-8, so basically we cross our fingers that the message force - # encoded to UTF-8 is readable. - message = response.commit_subject.dup.force_encoding('UTF-8') - - # NOTE: For ease of parsing in Gitaly, we have only the subject of - # the commit and not the full message. This is ok, since all the - # code that uses `local_branches` only cares at most about the - # commit message. - # TODO: Once gitaly "takes over" Rugged consider separating the - # subject from the message to make it clearer when there's one - # available but not the other. - hash = { - id: response.commit_id, - message: message, - authored_date: Time.at(response.commit_author.date.seconds), - author_name: response.commit_author.name, - author_email: response.commit_author.email, - committed_date: Time.at(response.commit_committer.date.seconds), - committer_name: response.commit_committer.name, - committer_email: response.commit_committer.email - } - - Gitlab::Git::Commit.decorate(hash) + def initialize(repository, name, target, target_commit) + super(repository, name, target, target_commit) end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 9c0606d780a..76a562f356e 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -38,7 +38,7 @@ module Gitlab repo = options.delete(:repo) raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) - repo.log(options).map { |c| decorate(c) } + repo.log(options) end # Get single commit @@ -48,6 +48,7 @@ module Gitlab # # Commit.find(repo, 'master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/321 def find(repo, commit_id = "HEAD") return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) @@ -97,7 +98,15 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.commits_between(base, head).map do |commit| + commits = Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| + if is_enabled + repo.gitaly_commit_client.between(base, head) + else + repo.commits_between(base, head) + end + end + + commits.map do |commit| decorate(commit) end rescue Rugged::ReferenceError @@ -124,6 +133,7 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) actual_options = options.dup @@ -208,6 +218,8 @@ module Gitlab init_from_hash(raw_commit) elsif raw_commit.is_a?(Rugged::Commit) init_from_rugged(raw_commit) + elsif raw_commit.is_a?(Gitaly::GitCommit) + init_from_gitaly(raw_commit) else raise "Invalid raw commit type: #{raw_commit.class}" end @@ -243,6 +255,8 @@ module Gitlab # Shows the diff between the commit's parent and the commit. # # Cuts out the header and stats from #to_patch and returns only the diff. + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/324 def to_diff diff_from_parent.patch end @@ -367,6 +381,22 @@ module Gitlab @parent_ids = commit.parents.map(&:oid) end + def init_from_gitaly(commit) + @raw_commit = commit + @id = commit.id + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + @message = (commit.body.presence || commit.subject).dup + @authored_date = Time.at(commit.author.date.seconds) + @author_name = commit.author.name.dup + @author_email = commit.author.email.dup + @committed_date = Time.at(commit.committer.date.seconds) + @committer_name = commit.committer.name.dup + @committer_email = commit.committer.email.dup + @parent_ids = commit.parent_ids + end + def serialize_keys SERIALIZE_KEYS end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index e9118bbed0e..57c29ad112c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: 1 RPC, migration in progress. + # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -6,6 +8,8 @@ module Gitlab attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(commit) @id = commit.id @additions = 0 diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 78e440395a5..7cb842256d0 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class Compare diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index cf7829a583b..cf95f673667 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. + # Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git @@ -81,110 +83,16 @@ module Gitlab # Return a copy of the +options+ hash containing only keys that can be # passed to Rugged. Allowed options are: # - # :max_size :: - # An integer specifying the maximum byte size of a file before a it - # will be treated as binary. The default value is 512MB. - # - # :context_lines :: - # The number of unchanged lines that define the boundary of a hunk - # (and to display before and after the actual changes). The default is - # 3. - # - # :interhunk_lines :: - # The maximum number of unchanged lines between hunk boundaries before - # the hunks will be merged into a one. The default is 0. - # - # :old_prefix :: - # The virtual "directory" to prefix to old filenames in hunk headers. - # The default is "a". - # - # :new_prefix :: - # The virtual "directory" to prefix to new filenames in hunk headers. - # The default is "b". - # - # :reverse :: - # If true, the sides of the diff will be reversed. - # - # :force_text :: - # If true, all files will be treated as text, disabling binary - # attributes & detection. - # - # :ignore_whitespace :: - # If true, all whitespace will be ignored. - # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :ignore_whitespace_eol :: - # If true, whitespace at end of line will be ignored. - # - # :ignore_submodules :: - # if true, submodules will be excluded from the diff completely. - # - # :patience :: - # If true, the "patience diff" algorithm will be used (currenlty - # unimplemented). - # - # :include_ignored :: - # If true, ignored files will be included in the diff. - # - # :include_untracked :: - # If true, untracked files will be included in the diff. - # - # :include_unmodified :: - # If true, unmodified files will be included in the diff. - # - # :recurse_untracked_dirs :: - # Even if +:include_untracked+ is true, untracked directories will - # only be marked with a single entry in the diff. If this flag is set - # to true, all files under ignored directories will be included in the - # diff, too. - # # :disable_pathspec_match :: # If true, the given +*paths+ will be applied as exact matches, # instead of as fnmatch patterns. # - # :deltas_are_icase :: - # If true, filename comparisons will be made with case-insensitivity. - # - # :include_untracked_content :: - # if true, untracked content will be contained in the the diff patch - # text. - # - # :skip_binary_check :: - # If true, diff deltas will be generated without spending time on - # binary detection. This is useful to improve performance in cases - # where the actual file content difference is not needed. - # - # :include_typechange :: - # If true, type changes for files will not be interpreted as deletion - # of the "old file" and addition of the "new file", but will generate - # typechange records. - # - # :include_typechange_trees :: - # Even if +:include_typechange+ is true, blob -> tree changes will - # still usually be handled as a deletion of the blob. If this flag is - # set to true, blob -> tree changes will be marked as typechanges. - # - # :ignore_filemode :: - # If true, file mode changes will be ignored. - # - # :recurse_ignored_dirs :: - # Even if +:include_ignored+ is true, ignored directories will only be - # marked with a single entry in the diff. If this flag is set to true, - # all files under ignored directories will be included in the diff, - # too. def filter_diff_options(options, default_options = {}) - allowed_options = [:max_size, :context_lines, :interhunk_lines, - :old_prefix, :new_prefix, :reverse, :force_text, - :ignore_whitespace, :ignore_whitespace_change, - :ignore_whitespace_eol, :ignore_submodules, - :patience, :include_ignored, :include_untracked, - :include_unmodified, :recurse_untracked_dirs, - :disable_pathspec_match, :deltas_are_icase, - :include_untracked_content, :skip_binary_check, - :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths, + allowed_options = [:ignore_whitespace_change, + :disable_pathspec_match, :paths, :max_files, :max_lines, :limits, :expanded] if default_options diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 555894907cc..0d8fe185ac5 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class DiffCollection diff --git a/lib/gitlab/git/env.rb b/lib/gitlab/git/env.rb index 0fdc57ec954..f80193ac553 100644 --- a/lib/gitlab/git/env.rb +++ b/lib/gitlab/git/env.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git # Ephemeral (per request) storage for environment variables that some Git diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index f4e3b5e5129..4a43b9b444d 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class GitmodulesParser diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 5042916343b..8f0c377ef4f 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: looks like this is only used by GitHooksService in +# app/services. We shouldn't bother migrating this until we know how +# GitHooksService will be migrated. + module Gitlab module Git class Hook diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index 666743006e5..db532600d1b 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,3 +1,7 @@ +# Gitaly note: JV: When the time comes I think we will want to copy this +# class into Gitaly. None of its methods look like they should be RPC's. +# The RPC's will be at a higher level. + module Gitlab module Git class Index diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 0148cd8df05..42c80aabd0a 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git class PathHelper diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index df9ca3ee5ac..25fa62ce4bd 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + require 'open3' module Gitlab diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index ebf7393dc61..372ce005b94 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). + module Gitlab module Git class Ref @@ -24,16 +26,16 @@ module Gitlab str.gsub(/\Arefs\/heads\//, '') end + # Gitaly: this method will probably be migrated indirectly via its call sites. def self.dereference_object(object) object = object.target while object.is_a?(Rugged::Tag::Annotation) object end - def initialize(repository, name, target) - encode! name - @name = name.gsub(/\Arefs\/(tags|heads)\//, '') - @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + def initialize(repository, name, target, derefenced_target) + @name = Gitlab::Git.ref_name(name) + @dereferenced_target = derefenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e51966313d4..63eebadff2e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -80,16 +80,10 @@ module Gitlab end # Returns an Array of Branches - def branches(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/389 + def branches(sort_by: nil) + branches_filter(sort_by: sort_by) end def reload_rugged @@ -107,7 +101,10 @@ module Gitlab reload_rugged if force_reload rugged_ref = rugged.branches[name] - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + if rugged_ref + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + end end def local_branches(sort_by: nil) @@ -115,7 +112,7 @@ module Gitlab if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else - branches(filter: :local, sort_by: sort_by) + branches_filter(filter: :local, sort_by: sort_by) end end end @@ -162,6 +159,8 @@ module Gitlab end # Returns an Array of Tags + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags rugged.references.each("refs/tags/*").map do |ref| message = nil @@ -174,7 +173,8 @@ module Gitlab end end - Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + target_commit = Gitlab::Git::Commit.find(self, ref.target) + Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) end.sort_by(&:name) end @@ -204,13 +204,6 @@ module Gitlab branch_names + tag_names end - # Deprecated. Will be removed in 5.2 - def heads - rugged.references.each("refs/heads/*").map do |head| - Gitlab::Git::Ref.new(self, head.name, head.target) - end.sort_by(&:name) - end - def has_commits? !empty? end @@ -297,28 +290,6 @@ module Gitlab (size.to_f / 1024).round(2) end - # Returns an array of BlobSnippets for files at the specified +ref+ that - # contain the +query+ string. - def search_files(query, ref = nil) - greps = [] - ref ||= root_ref - - populated_index(ref).each do |entry| - # Discard submodules - next if submodule?(entry) - - blob = Gitlab::Git::Blob.raw(self, entry[:oid]) - - # Skip binary files - next if blob.data.encoding == Encoding::ASCII_8BIT - - blob.load_all_data!(self) - greps += build_greps(blob.data, query, ref, entry[:path]) - end - - greps - end - # Use the Rugged Walker API to build an array of commits. # # Usage. @@ -331,85 +302,10 @@ module Gitlab # ) # def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 - actual_ref = options[:ref] || root_ref - begin - sha = sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end - end - - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << sha - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output = IO.popen(cmd) { |io| io.read } - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + raw_log(options).map { |c| Commit.decorate(c) } end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/382 def count_commits(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] @@ -454,7 +350,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - commits_between(from, to).size + Commit.between(self, from, to).size end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -553,6 +449,7 @@ module Gitlab # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| if is_enabled @@ -567,6 +464,8 @@ module Gitlab end # Return total commits count accessible from passed ref + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) gitaly_migrate(:commit_count) do |is_enabled| if is_enabled @@ -779,7 +678,8 @@ module Gitlab # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") rugged_ref = rugged.branches.create(ref, start_point) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ raise InvalidRef.new("Invalid reference #{start_point}") @@ -838,6 +738,7 @@ module Gitlab # Ex. # repo.ls_files('master') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) actual_ref = ref || root_ref @@ -864,6 +765,7 @@ module Gitlab raw_output.compact end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) begin commit = lookup(ref) @@ -905,8 +807,113 @@ module Gitlab Gitlab::GitalyClient::Util.repository(@storage, @relative_path) end + def gitaly_ref_client + @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) + end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) + end + private + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. + def branches_filter(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact + + sort_branches(branches, sort_by) + end + + def raw_log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_NONE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + # Gitaly note: JV: although #log_by_shell shells out to Git I think the + # complexity is such that we should migrate it as Ruby before trying to + # do it in Go. + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) + end + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + # We are trying to deprecate this method because it does a lot of work # but it seems to be used only to look up submodule URL's. # https://gitlab.com/gitlab-org/gitaly/issues/329 @@ -930,7 +937,7 @@ module Gitlab return unless commit_object && commit_object.type == :COMMIT - gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Blob::MAX_DATA_DISPLAY_SIZE) + gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) found_module = GitmodulesParser.new(gitmodules.data).parse[path] found_module && found_module['url'] @@ -1078,73 +1085,6 @@ module Gitlab index end - # Return an array of BlobSnippets for lines in +file_contents+ that match - # +query+ - def build_greps(file_contents, query, ref, filename) - # The file_contents string is potentially huge so we make sure to loop - # through it one line at a time. This gives Ruby the chance to GC lines - # we are not interested in. - # - # We need to do a little extra work because we are not looking for just - # the lines that matches the query, but also for the context - # (surrounding lines). We will use Enumerable#each_cons to efficiently - # loop through the lines while keeping surrounding lines on hand. - # - # First, we turn "foo\nbar\nbaz" into - # [ - # [nil, -3], [nil, -2], [nil, -1], - # ['foo', 0], ['bar', 1], ['baz', 3], - # [nil, 4], [nil, 5], [nil, 6] - # ] - lines_with_index = Enumerator.new do |yielder| - # Yield fake 'before' lines for the first line of file_contents - (-SEARCH_CONTEXT_LINES..-1).each do |i| - yielder.yield [nil, i] - end - - # Yield the actual file contents - count = 0 - file_contents.each_line do |line| - line.chomp! - yielder.yield [line, count] - count += 1 - end - - # Yield fake 'after' lines for the last line of file_contents - (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| - yielder.yield [nil, i] - end - end - - greps = [] - - # Loop through consecutive blocks of lines with indexes - lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| - # Get the 'middle' line and index from the block - line, _ = line_block[SEARCH_CONTEXT_LINES] - - next unless line && line.match(/#{Regexp.escape(query)}/i) - - # Yay, 'line' contains a match! - # Get an array with just the context lines (no indexes) - match_with_context = line_block.map(&:first) - # Remove 'nil' lines in case we are close to the first or last line - match_with_context.compact! - - # Get the line number (1-indexed) of the first context line - first_context_line_number = line_block[0][1] + 1 - - greps << Gitlab::Git::BlobSnippet.new( - ref, - match_with_context, - first_context_line_number, - filename - ) - end - - greps - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} @@ -1173,14 +1113,6 @@ module Gitlab end end - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) - end - - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) - end - def gitaly_migrate(method, &block) Gitlab::GitalyClient.migrate(method, &block) rescue GRPC::NotFound => e diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index a16b0ed76f4..2b5785a1f08 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. + module Gitlab module Git class RevList @@ -15,6 +17,8 @@ module Gitlab end # This methods returns an array of missed references + # + # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index b5342c3d310..bc4e160dce9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,10 +1,12 @@ +# Gitaly note: JV: no RPC's here. +# module Gitlab module Git class Tag < Ref attr_reader :object_sha - def initialize(repository, name, target, message = nil) - super(repository, name, target) + def initialize(repository, name, target, target_commit, message = nil) + super(repository, name, target, target_commit) @message = message end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b6d4e6cfe46..8122ff0e81f 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: needs 1 RPC, migration is in progress. + module Gitlab module Git class Tree @@ -10,6 +12,8 @@ module Gitlab # Get list of tree objects # for repository based on commit sha and path # Uses rugged for raw objects + # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where(repository, sha, path = nil) path = nil if path == '' || path == '/' @@ -40,6 +44,8 @@ module Gitlab end end + private + # Recursive search of tree id for path # # Ex. diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 7973da2e8f8..4708f22dcb3 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: no RPC's here. + module Gitlab module Git module Util diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 0e87ee30c98..a3c6b21a6a1 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# Gitaly note: JV: does not need to be migrated, works without a repo. + module Gitlab module GitRefValidator extend self diff --git a/lib/gitlab/gitaly_client/blob.rb b/lib/gitlab/gitaly_client/blob_service.rb index 0c398b46a08..7ea8e8d0857 100644 --- a/lib/gitlab/gitaly_client/blob.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,10 +1,10 @@ module Gitlab module GitalyClient - class Blob + class BlobService def initialize(repository) @gitaly_repo = repository.gitaly_repository end - + def get_blob(oid:, limit:) request = Gitaly::GetBlobRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit_service.rb index aafc0520664..8f5738fed06 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Commit + class CommitService # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -17,20 +17,20 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit, :commit_is_ancestor, request).value + GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request).value end def diff_from_parent(commit, options = {}) request_params = commit_diff_request_params(commit, options) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff, :commit_diff, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request) Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(commit_diff_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff, :commit_delta, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) response.flat_map do |msg| msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end @@ -44,7 +44,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit, :tree_entry, request) + response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request) entry = response.first return unless entry.oid.present? @@ -65,6 +65,17 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request).count end + def between(from, to) + request = Gitaly::CommitsBetweenRequest.new( + repository: @gitaly_repo, + from: from, + to: to + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request) + consume_commits_response(response) + end + private def commit_diff_request_params(commit, options = {}) @@ -77,6 +88,10 @@ module Gitlab paths: options.fetch(:paths, []) } end + + def consume_commits_response(response) + response.flat_map { |r| r.commits } + end end end end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notification_service.rb index 78ed433e6b8..326e6f7dafc 100644 --- a/lib/gitlab/gitaly_client/notifications.rb +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Notifications + class NotificationService # 'repository' is a Gitlab::Git::Repository def initialize(repository) @gitaly_repo = repository.gitaly_repository @@ -10,7 +10,7 @@ module Gitlab def post_receive GitalyClient.call( @storage, - :notifications, + :notification_service, :post_receive, Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) ) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref_service.rb index 6edc69de078..2c3d53410ac 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -1,6 +1,6 @@ module Gitlab module GitalyClient - class Ref + class RefService include Gitlab::EncodingHelper # 'repository' is a Gitlab::Git::Repository @@ -12,19 +12,19 @@ module Gitlab def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_default_branch_name, request) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_branch_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref, :find_all_tag_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -34,7 +34,7 @@ module Gitlab commit_id: commit_id, prefix: ref_prefix ) - encode!(GitalyClient.call(@storage, :ref, :find_ref_name, request).name.dup) + encode!(GitalyClient.call(@storage, :ref_service, :find_ref_name, request).name.dup) end def count_tag_names @@ -48,7 +48,7 @@ module Gitlab def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref, :find_local_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) consume_branches_response(response) end @@ -72,11 +72,39 @@ module Gitlab Gitlab::Git::Branch.new( @repository, encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) ) end end end + + def commit_from_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name.dup, + author_email: response.commit_author.email.dup, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name.dup, + committer_email: response.commit_committer.email.dup + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 70da4080cae..bebde857b16 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -35,9 +35,9 @@ module Gitlab repository_storages.flat_map do |storage_name| tmp_file_path = tmp_file_path(storage_name) [ - operation_metrics(:filesystem_accessible, :filesystem_access_latency, -> { storage_stat_test(storage_name) }, shard: storage_name), - operation_metrics(:filesystem_writable, :filesystem_write_latency, -> { storage_write_test(tmp_file_path) }, shard: storage_name), - operation_metrics(:filesystem_readable, :filesystem_read_latency, -> { storage_read_test(tmp_file_path) }, shard: storage_name) + operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, -> { storage_stat_test(storage_name) }, shard: storage_name), + operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, -> { storage_write_test(tmp_file_path) }, shard: storage_name), + operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, -> { storage_read_test(tmp_file_path) }, shard: storage_name) ].flatten end end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb new file mode 100644 index 00000000000..a28658d42d4 --- /dev/null +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class CacheCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_cache_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Cache.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb new file mode 100644 index 00000000000..f97d50d3947 --- /dev/null +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class QueuesCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_queues_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::Queues.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb new file mode 100644 index 00000000000..fe4e3c4a3ab --- /dev/null +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -0,0 +1,27 @@ +module Gitlab + module HealthChecks + module Redis + class RedisCheck + extend SimpleAbstractCheck + + class << self + private + + def metric_prefix + 'redis_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && + ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb new file mode 100644 index 00000000000..e3244392902 --- /dev/null +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -0,0 +1,31 @@ +module Gitlab + module HealthChecks + module Redis + class SharedStateCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_shared_state_ping' + end + + def is_successful?(result) + result == 'PONG' + end + + def check + catch_timeout 10.seconds do + Gitlab::Redis::SharedState.with(&:ping) + end + end + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb deleted file mode 100644 index 57bbe5b3ad0..00000000000 --- a/lib/gitlab/health_checks/redis_check.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module HealthChecks - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def is_successful?(result) - result == 'PONG' - end - - def check - catch_timeout 10.seconds do - Gitlab::Redis.with(&:ping) - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index fbe1645c1b1..3dcb28a193c 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -20,7 +20,7 @@ module Gitlab [ metric("#{metric_prefix}_timeout", result.is_a?(Timeout::Error) ? 1 : 0), metric("#{metric_prefix}_success", is_successful?(result) ? 1 : 0), - metric("#{metric_prefix}_latency", elapsed) + metric("#{metric_prefix}_latency_seconds", elapsed) ] end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f3d489aad0d..a1b896c9511 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -12,8 +12,11 @@ module Gitlab 'zh_HK' => '繁體中文(香港)', 'zh_TW' => '繁體中文(臺灣)', 'bg' => 'български', + 'ru' => 'Русский', 'eo' => 'Esperanto', - 'it' => 'Italiano' + 'it' => 'Italiano', + 'uk' => 'Українська', + 'ja' => '日本語' }.freeze def available_locales diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb new file mode 100644 index 00000000000..977c05910d3 --- /dev/null +++ b/lib/gitlab/issuable_metadata.rb @@ -0,0 +1,36 @@ +module Gitlab + module IssuableMetadata + def issuable_meta_data(issuable_collection, collection_type) + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_ids = issuable_collection.map(&:id) + + return {} if issuable_ids.empty? + + issuable_note_count = ::Note.count_for_collection(issuable_ids, collection_type) + issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) + issuable_merge_requests_count = + if collection_type == 'Issue' + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids) + else + [] + end + + issuable_ids.each_with_object({}) do |id, issuable_meta| + downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = issuable_note_count.find { |notes| notes.noteable_id == id } + merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } + + issuable_meta[id] = ::Issuable::IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + end + end +end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 5f67e97fa2a..8e57ba831c5 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -18,10 +18,10 @@ module Gitlab end def token - Gitlab::Redis.with do |redis| - token = redis.get(redis_key) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key) token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_key, token, ex: EXPIRY_TIME) + redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) token end @@ -41,7 +41,7 @@ module Gitlab private - def redis_key + def redis_shared_state_key "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 3503fac40e8..9f432673a6e 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,6 +1,6 @@ require 'yaml' require 'json' -require_relative 'redis' unless defined?(Gitlab::Redis) +require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) module Gitlab module MailRoom @@ -34,11 +34,11 @@ module Gitlab config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - gitlab_redis = Gitlab::Redis.new(rails_env) - config[:redis_url] = gitlab_redis.url + gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) + config[:redis_url] = gitlab_redis_queues.url - if gitlab_redis.sentinels? - config[:sentinels] = gitlab_redis.sentinels + if gitlab_redis_queues.sentinels? + config[:sentinels] = gitlab_redis_queues.sentinels end end diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb deleted file mode 100644 index b3da360be8f..00000000000 --- a/lib/gitlab/metrics/connection_rack_middleware.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Gitlab - module Metrics - class ConnectionRackMiddleware - def initialize(app) - @app = app - end - - def self.rack_request_count - @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count') - end - - def self.rack_response_count - @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count') - end - - def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count') - end - - def self.rack_execution_time - @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10]) - end - - def call(env) - method = env['REQUEST_METHOD'].downcase - started = Time.now.to_f - begin - ConnectionRackMiddleware.rack_request_count.increment(method: method) - - status, headers, body = @app.call(env) - - ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status) - [status, headers, body] - rescue - ConnectionRackMiddleware.rack_uncaught_errors_count.increment - raise - ensure - elapsed = Time.now.to_f - started - ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed) - end - end - end - end -end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index fb7bbc7cfc7..460dab47276 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -6,9 +6,11 @@ module Gitlab include Gitlab::CurrentSettings def metrics_folder_present? - ENV.has_key?('prometheus_multiproc_dir') && - ::Dir.exist?(ENV['prometheus_multiproc_dir']) && - ::File.writable?(ENV['prometheus_multiproc_dir']) + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) end def prometheus_metrics_enabled? diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb new file mode 100644 index 00000000000..0dc19f31d03 --- /dev/null +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -0,0 +1,40 @@ +module Gitlab + module Metrics + class RequestsRackMiddleware + def initialize(app) + @app = app + end + + def self.http_request_total + @http_request_total ||= Gitlab::Metrics.counter(:http_requests_total, 'Request count') + end + + def self.rack_uncaught_errors_count + @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + end + + def self.http_request_duration_seconds + @http_request_duration_seconds ||= Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + end + + def call(env) + method = env['REQUEST_METHOD'].downcase + started = Time.now.to_f + begin + RequestsRackMiddleware.http_request_total.increment(method: method) + + status, headers, body = @app.call(env) + + elapsed = Time.now.to_f - started + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status }, elapsed) + + [status, headers, body] + rescue + RequestsRackMiddleware.rack_uncaught_errors_count.increment + raise + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index b3f453e506d..3f2bbd9f6a6 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -101,14 +101,18 @@ module Gitlab # Look for a corresponding person with same uid in any of the configured LDAP providers Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) - @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) - # The `uid` might actually be a DN. Try it next. - @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + @ldap_person = find_ldap_person(auth_hash, adapter) break if @ldap_person end @ldap_person end + def find_ldap_person(auth_hash, adapter) + by_uid = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + by_uid || Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + end + def ldap_config Gitlab::LDAP::Config.new(ldap_person.provider) if ldap_person end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index d81f825ef96..60a32d5d5ea 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -49,7 +49,6 @@ module Gitlab sent_notifications services snippets - system teams u unicorn_test diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 2da2ce45ebc..56112ec2301 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -2,7 +2,8 @@ module Gitlab module PerformanceBar include Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids'.freeze + ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze + EXPIRY_TIME = 5.minutes def self.enabled?(user = nil) return false unless user && allowed_group_id @@ -15,7 +16,7 @@ module Gitlab end def self.allowed_user_ids - Rails.cache.fetch(ALLOWED_USER_IDS_KEY) do + Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) if group diff --git a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb b/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb deleted file mode 100644 index d939a6ea18d..00000000000 --- a/lib/gitlab/performance_bar/peek_performance_bar_with_rack_body.rb +++ /dev/null @@ -1,22 +0,0 @@ -# This solves a bug with a X-Senfile header that wouldn't be set properly, see -# https://github.com/peek/peek-performance_bar/pull/27 -module Gitlab - module PerformanceBar - module PeekPerformanceBarWithRackBody - def call(env) - @env = env - reset_stats - - @total_requests += 1 - first_request if @total_requests == 1 - - env['process.request_start'] = @start.to_f - env['process.total_requests'] = total_requests - - status, headers, body = @app.call(env) - body = Rack::BodyProxy.new(body) { record_request } - [status, headers, body] - end - end - end -end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index 574ae8731a5..67fee8c227d 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -1,4 +1,5 @@ # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb +# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb module Gitlab module PerformanceBar module PeekQueryTracker @@ -23,14 +24,20 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if RequestStore.active? && RequestStore.store[:peek_enabled] - track_query(data[:sql].strip, data[:binds], start, finish) + # data[:cached] is only available starting from Rails 5.1.0 + # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 + # Before that, data[:name] was set to 'CACHE' + # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 + unless data.fetch(:cached, data[:name] == 'CACHE') + track_query(data[:sql].strip, data[:binds], start, finish) + end end end end def track_query(raw_query, bindings, start, finish) query = Gitlab::Sherlock::Query.new(raw_query, start, finish) - query_info = { duration: '%.3f' % query.duration, sql: query.formatted_query } + query_info = { duration: query.duration.round(3), sql: query.formatted_query } PEEK_DB_CLIENT.query_details << query_info end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb deleted file mode 100644 index bc5370de32a..00000000000 --- a/lib/gitlab/redis.rb +++ /dev/null @@ -1,102 +0,0 @@ -# This file should not have any direct dependency on Rails environment -# please require all dependencies below: -require 'active_support/core_ext/hash/keys' -require 'active_support/core_ext/module/delegation' - -module Gitlab - class Redis - CACHE_NAMESPACE = 'cache:gitlab'.freeze - SESSION_NAMESPACE = 'session:gitlab'.freeze - SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze - MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze - DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze - - class << self - delegate :params, :url, to: :new - - def with - @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } - @pool.with { |redis| yield redis } - end - - def pool_size - if Sidekiq.server? - # the pool will be used in a multi-threaded context - Sidekiq.options[:concurrency] + 5 - else - # probably this is a Unicorn process, so single threaded - 5 - end - end - - def _raw_config - return @_raw_config if defined?(@_raw_config) - - begin - @_raw_config = ERB.new(File.read(config_file)).result.freeze - rescue Errno::ENOENT - @_raw_config = false - end - - @_raw_config - end - - def config_file - ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) - end - end - - def initialize(rails_env = nil) - @rails_env = rails_env || ::Rails.env - end - - def params - redis_store_options - end - - def url - raw_config_hash[:url] - end - - def sentinels - raw_config_hash[:sentinels] - end - - def sentinels? - sentinels && !sentinels.empty? - end - - private - - def redis_store_options - config = raw_config_hash - redis_url = config.delete(:url) - redis_uri = URI.parse(redis_url) - - if redis_uri.scheme == 'unix' - # Redis::Store does not handle Unix sockets well, so let's do it for them - config[:path] = redis_uri.path - config - else - redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) - # order is important here, sentinels must be after the connection keys. - # {url: ..., port: ..., sentinels: [...]} - redis_hash.merge(config) - end - end - - def raw_config_hash - config_data = fetch_config - - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: DEFAULT_REDIS_URL } - end - end - - def fetch_config - self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false - end - end -end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb new file mode 100644 index 00000000000..b0da516ff83 --- /dev/null +++ b/lib/gitlab/redis/cache.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Cache < ::Gitlab::Redis::Wrapper + CACHE_NAMESPACE = 'cache:gitlab'.freeze + DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze + REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_CACHE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb new file mode 100644 index 00000000000..f9249d05565 --- /dev/null +++ b/lib/gitlab/redis/queues.rb @@ -0,0 +1,35 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class Queues < ::Gitlab::Redis::Wrapper + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze + REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_QUEUES_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb new file mode 100644 index 00000000000..395dcf082da --- /dev/null +++ b/lib/gitlab/redis/shared_state.rb @@ -0,0 +1,34 @@ +# please require all dependencies below: +require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) + +module Gitlab + module Redis + class SharedState < ::Gitlab::Redis::Wrapper + SESSION_NAMESPACE = 'session:gitlab'.freeze + DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze + REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze + end + + class << self + def default_url + DEFAULT_REDIS_SHARED_STATE_URL + end + + def config_file_name + # if ENV set for this class, use it even if it points to a file does not exist + file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME] + return file_name if file_name + + # otherwise, if config files exists for this class, use it + file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent + super + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb new file mode 100644 index 00000000000..c43b37dde74 --- /dev/null +++ b/lib/gitlab/redis/wrapper.rb @@ -0,0 +1,135 @@ +# This file should only be used by sub-classes, not directly by any clients of the sub-classes +# please require all dependencies below: +require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' + +module Gitlab + module Redis + class Wrapper + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze + if defined?(::Rails) && ::Rails.root.present? + DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze + end + + class << self + delegate :params, :url, to: :new + + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } + end + + def pool_size + # heuristic constant 5 should be a config setting somewhere -- related to CPU count? + size = 5 + if Sidekiq.server? + # the pool will be used in a multi-threaded context + size += Sidekiq.options[:concurrency] + end + size + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + @_raw_config = + begin + if filename = config_file_name + ERB.new(File.read(filename)).result.freeze + else + false + end + rescue Errno::ENOENT + false + end + end + + def default_url + DEFAULT_REDIS_URL + end + + def config_file_name + # if ENV set for wrapper class, use it even if it points to a file does not exist + file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME] + return file_name unless file_name.nil? + + # otherwise, if config files exists for wrapper class, use it + file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__) + return file_name if File.file?(file_name) + + # nil will force use of DEFAULT_REDIS_URL when config file is absent + nil + end + end + + def initialize(rails_env = nil) + @rails_env = rails_env || ::Rails.env + end + + def params + redis_store_options + end + + def url + raw_config_hash[:url] + end + + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + + private + + def redis_store_options + config = raw_config_hash + redis_url = config.delete(:url) + redis_uri = URI.parse(redis_url) + + if redis_uri.scheme == 'unix' + # Redis::Store does not handle Unix sockets well, so let's do it for them + config[:path] = redis_uri.path + query = redis_uri.query + unless query.nil? + queries = CGI.parse(redis_uri.query) + db_numbers = queries["db"] if queries.key?("db") + config[:db] = db_numbers[0].to_i if db_numbers.any? + end + config + else + redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + # order is important here, sentinels must be after the connection keys. + # {url: ..., port: ..., sentinels: [...]} + redis_hash.merge(config) + end + end + + def raw_config_hash + config_data = fetch_config + + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: self.class.default_url } + end + end + + def fetch_config + return false unless self.class._raw_config + + yaml = YAML.load(self.class._raw_config) + + # If the file has content but it's invalid YAML, `load` returns false + if yaml + yaml.fetch(@rails_env, false) + else + false + end + end + end + end +end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 877aa6e6a28..f3952657983 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -18,7 +18,11 @@ module Gitlab mapping = @map.find { |mapping| mapping[:source] === path } return unless mapping - path.sub(mapping[:source], mapping[:public]) + if mapping[:source].is_a?(String) + path.sub(mapping[:source], mapping[:public]) + else + mapping[:source].replace(path, mapping[:public]) + end end private @@ -35,7 +39,7 @@ module Gitlab source_pattern = source_pattern[1...-1].gsub('\/', '/') begin - source_pattern = /\A#{source_pattern}\z/ + source_pattern = Gitlab::UntrustedRegexp.new('\A' + source_pattern + '\z') rescue RegexpError => e raise FormatError, "Route map entry source is not a valid regular expression: #{e}" end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 0baea092e6a..4366ff336ef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,3 +1,6 @@ +# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. +# SSH key operations are not part of Gitaly so will never be migrated. + require 'securerandom' module Gitlab @@ -68,6 +71,7 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'add-project', storage, "#{name}.git"]) @@ -81,6 +85,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def import_repository(storage, name, url) # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. @@ -99,6 +104,7 @@ module Gitlab # Ex. # fetch_remote("gitlab/gitlab-ci", "upstream") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(storage, name, remote, forced: false, no_tags: false) args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced @@ -115,6 +121,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def mv_repository(storage, path, new_path) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project', storage, "#{path}.git", "#{new_path}.git"]) @@ -129,6 +136,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "randx") # + # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. def fork_repository(forked_from_storage, path, forked_to_storage, fork_namespace) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'fork-project', forked_from_storage, "#{path}.git", forked_to_storage, @@ -143,6 +151,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def remove_repository(storage, name) gitlab_shell_fast_execute([gitlab_shell_projects_path, 'rm-project', storage, "#{name}.git"]) @@ -194,6 +203,7 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) path = full_path(storage, name) FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) @@ -207,6 +217,7 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) FileUtils.rm_r(full_path(storage, name), force: true) end @@ -216,6 +227,7 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) return false if exists?(storage, new_name) || !exists?(storage, old_name) @@ -241,6 +253,7 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) File.exist?(full_path(storage, dir_name)) end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb new file mode 100644 index 00000000000..8b43f0053d6 --- /dev/null +++ b/lib/gitlab/untrusted_regexp.rb @@ -0,0 +1,53 @@ +module Gitlab + # An untrusted regular expression is any regexp containing patterns sourced + # from user input. + # + # Ruby's built-in regular expression library allows patterns which complete in + # exponential time, permitting denial-of-service attacks. + # + # Not all regular expression features are available in untrusted regexes, and + # there is a strict limit on total execution time. See the RE2 documentation + # at https://github.com/google/re2/wiki/Syntax for more details. + class UntrustedRegexp + delegate :===, to: :regexp + + def initialize(pattern) + @regexp = RE2::Regexp.new(pattern, log_errors: false) + + raise RegexpError.new(regexp.error) unless regexp.ok? + end + + def replace_all(text, rewrite) + RE2.GlobalReplace(text, regexp, rewrite) + end + + def scan(text) + scan_regexp.scan(text).map do |match| + if regexp.number_of_capturing_groups == 0 + match.first + else + match + end + end + end + + def replace(text, rewrite) + RE2.Replace(text, regexp, rewrite) + end + + private + + attr_reader :regexp + + # RE2 scan operates differently to Ruby scan when there are no capture + # groups, so work around it + def scan_regexp + @scan_regexp ||= + if regexp.number_of_capturing_groups == 0 + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 35792d2d67f..824e2d7251f 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -52,15 +52,13 @@ module Gitlab commit_url(id: object.commit_id, anchor: dom_id(object)) elsif object.for_issue? - issue = Issue.find(object.noteable_id) - issue_url(issue, anchor: dom_id(object)) + issue_url(object.noteable, anchor: dom_id(object)) elsif object.for_merge_request? - merge_request = MergeRequest.find(object.noteable_id) - merge_request_url(merge_request, anchor: dom_id(object)) + merge_request_url(object.noteable, anchor: dom_id(object)) elsif object.for_snippet? - snippet = Snippet.find(object.noteable_id) + snippet = object.noteable if snippet.is_a?(PersonalSnippet) snippet_url(snippet, anchor: dom_id(object)) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f19b325a126..dba071d7e47 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -39,6 +39,7 @@ module Gitlab notes: Note.count, pages_domains: PagesDomain.count, projects: Project.count, + projects_imported_from_github: Project.where(import_type: 'github').count, projects_prometheus_active: PrometheusService.active.count, protected_branches: ProtectedBranch.count, releases: Release.count, diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 3b922da7ced..8e91ee7287c 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,5 +1,11 @@ module Gitlab class UserAccess + extend Gitlab::Cache::RequestCache + + request_cache_key do + [user&.id, project&.id] + end + attr_reader :user, :project def initialize(user, project: nil) @@ -28,7 +34,7 @@ module Gitlab true end - def can_create_tag?(ref) + request_cache def can_create_tag?(ref) return false unless can_access_git? if ProtectedTag.protected?(project, ref) @@ -38,7 +44,7 @@ module Gitlab end end - def can_delete_branch?(ref) + request_cache def can_delete_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) @@ -48,7 +54,7 @@ module Gitlab end end - def can_push_to_branch?(ref) + request_cache def can_push_to_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) @@ -60,7 +66,7 @@ module Gitlab end end - def can_merge_to_branch?(ref) + request_cache def can_merge_to_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb index eb36ab9fded..125488536e1 100644 --- a/lib/gitlab/user_activities.rb +++ b/lib/gitlab/user_activities.rb @@ -6,13 +6,13 @@ module Gitlab BATCH_SIZE = 500 def self.record(key, time = Time.now) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hset(KEY, key, time.to_i) end end def delete(*keys) - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hdel(KEY, keys) end end @@ -21,7 +21,7 @@ module Gitlab cursor = 0 loop do cursor, pairs = - Gitlab::Redis.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.hscan(KEY, cursor, count: BATCH_SIZE) end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 48f3d950779..c60bd91ea6e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -89,12 +89,12 @@ module Gitlab end def level_name(level) - level_name = 'Unknown' + level_name = N_('VisibilityLevel|Unknown') options.each do |name, lvl| level_name = name if lvl == level.to_i end - level_name + s_(level_name) end def level_value(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 4aef23b6aee..916ef365d78 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -62,10 +62,21 @@ module Gitlab end def send_git_blob(repository, blob) - params = { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } + params = if Gitlab::GitalyClient.feature_enabled?(:project_raw_show) + { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } + else + { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id + } + end [ SEND_DATA_HEADER, @@ -176,7 +187,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis.with do |redis| + Gitlab::Redis::Queues.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") @@ -192,6 +203,13 @@ module Gitlab def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + + def gitaly_server_hash(repository) + { + address: Gitlab::GitalyClient.address(repository.project.repository_storage), + token: Gitlab::GitalyClient.token(repository.project.repository_storage) + } + end end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 125a3d560d6..564aa141952 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -5,12 +5,12 @@ namespace :cache do desc "GitLab | Clear redis cache" task redis: :environment do - Gitlab::Redis.with do |redis| + Gitlab::Redis::Cache.with do |redis| cursor = REDIS_SCAN_START_STOP loop do cursor, keys = redis.scan( cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*", count: REDIS_CLEAR_BATCH_SIZE ) diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index b27f7475115..b48e4dce445 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -5,7 +5,7 @@ namespace :gettext do # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate folders = %W(app lib config #{locale_path}).join(',') - exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( "{#{folders}}/**/*.{#{exts}}" |