diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-23 00:08:53 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-01-23 00:08:53 +0000 |
commit | d65442b1d9621da6749d59ea1a544a2ea39b3a79 (patch) | |
tree | 0aa6fb67b70c38cdd949a2b4002ceb459860fa82 /lib/gitlab/database | |
parent | b6ec12ceca58b12d974d46d0579742f4d3cdb9d7 (diff) | |
download | gitlab-ce-d65442b1d9621da6749d59ea1a544a2ea39b3a79.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab/database')
-rw-r--r-- | lib/gitlab/database/migration_helpers.rb | 40 | ||||
-rw-r--r-- | lib/gitlab/database/with_lock_retries.rb | 158 |
2 files changed, 198 insertions, 0 deletions
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index b7d510c19f9..5077143e15e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -280,6 +280,46 @@ module Gitlab end end + # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. + # The timings can be controlled via the +timing_configuration+ parameter. + # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. + # + # ==== Examples + # # Invoking without parameters + # with_lock_retries do + # drop_table :my_table + # end + # + # # Invoking with custom +timing_configuration+ + # t = [ + # [1.second, 1.second], + # [2.seconds, 2.seconds] + # ] + # + # with_lock_retries(timing_configuration: t) do + # drop_table :my_table # this will be retried twice + # end + # + # # Disabling the retries using an environment variable + # > export DISABLE_LOCK_RETRIES=true + # + # with_lock_retries do + # drop_table :my_table # one invocation, it will not retry at all + # end + # + # ==== Parameters + # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` + # * +logger+ - [Gitlab::JsonLogger] + # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` + def with_lock_retries(**args, &block) + merged_args = { + klass: self.class, + logger: Gitlab::BackgroundMigration::Logger + }.merge(args) + + Gitlab::Database::WithLockRetries.new(merged_args).run(&block) + end + def true_value Database.true_value end diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb new file mode 100644 index 00000000000..37f7e8fbdac --- /dev/null +++ b/lib/gitlab/database/with_lock_retries.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class WithLockRetries + NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null') + + # Each element of the array represents a retry iteration. + # - DEFAULT_TIMING_CONFIGURATION.size provides the iteration count. + # - First element: DB lock_timeout + # - Second element: Sleep time after unsuccessful lock attempt (LockWaitTimeout error raised) + # - Worst case, this configuration would retry for about 40 minutes. + DEFAULT_TIMING_CONFIGURATION = [ + [0.1.seconds, 0.05.seconds], # short timings, lock_timeout: 100ms, sleep after LockWaitTimeout: 50ms + [0.1.seconds, 0.05.seconds], + [0.2.seconds, 0.05.seconds], + [0.3.seconds, 0.10.seconds], + [0.4.seconds, 0.15.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [1.second, 5.seconds], # probably high traffic, increase timings + [1.second, 1.minute], + [0.1.seconds, 0.05.seconds], + [0.1.seconds, 0.05.seconds], + [0.2.seconds, 0.05.seconds], + [0.3.seconds, 0.10.seconds], + [0.4.seconds, 0.15.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [3.seconds, 3.minutes], # probably high traffic or long locks, increase timings + [0.1.seconds, 0.05.seconds], + [0.1.seconds, 0.05.seconds], + [0.5.seconds, 2.seconds], + [0.5.seconds, 2.seconds], + [5.seconds, 2.minutes], + [0.5.seconds, 0.5.seconds], + [0.5.seconds, 0.5.seconds], + [7.seconds, 5.minutes], + [0.5.seconds, 0.5.seconds], + [0.5.seconds, 0.5.seconds], + [7.seconds, 5.minutes], + [0.5.seconds, 0.5.seconds], + [0.5.seconds, 0.5.seconds], + [7.seconds, 5.minutes], + [0.1.seconds, 0.05.seconds], + [0.1.seconds, 0.05.seconds], + [0.5.seconds, 2.seconds], + [10.seconds, 10.minutes], + [0.1.seconds, 0.05.seconds], + [0.5.seconds, 2.seconds], + [10.seconds, 10.minutes] + ].freeze + + def initialize(logger: NULL_LOGGER, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV) + @logger = logger + @klass = klass + @timing_configuration = timing_configuration + @env = env + @current_iteration = 1 + @log_params = { method: 'with_lock_retries', class: klass.to_s } + end + + def run(&block) + raise 'no block given' unless block_given? + + @block = block + + if lock_retries_disabled? + log(message: 'DISABLE_LOCK_RETRIES environment variable is true, executing the block without retry') + + return run_block + end + + begin + run_block_with_transaction + rescue ActiveRecord::LockWaitTimeout + if retry_with_lock_timeout? + wait_until_next_retry + + retry + else + run_block_without_lock_timeout + end + end + end + + private + + attr_reader :logger, :env, :block, :current_iteration, :log_params, :timing_configuration + + def run_block + block.call + end + + def run_block_with_transaction + ActiveRecord::Base.transaction(requires_new: true) do + execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'") + + log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + end + end + + def retry_with_lock_timeout? + current_iteration != retry_count + end + + def wait_until_next_retry + log(message: 'ActiveRecord::LockWaitTimeout error, retrying after sleep', current_iteration: current_iteration, sleep_time_in_seconds: current_sleep_time_in_seconds) + + sleep(current_sleep_time_in_seconds) + + @current_iteration += 1 + end + + def run_block_without_lock_timeout + log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) + log(message: "Executing the migration without lock timeout", current_iteration: current_iteration) + + execute("SET LOCAL lock_timeout TO '0'") + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration) + end + + def lock_retries_disabled? + Gitlab::Utils.to_boolean(env['DISABLE_LOCK_RETRIES']) + end + + def log(params) + logger.info(log_params.merge(params)) + end + + def execute(statement) + ActiveRecord::Base.connection.execute(statement) + end + + def retry_count + timing_configuration.size + end + + def current_lock_timeout_in_ms + timing_configuration[current_iteration - 1][0].in_milliseconds + end + + def current_sleep_time_in_seconds + timing_configuration[current_iteration - 1][1].to_i + end + end + end +end |