diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/circuit_breakers.rb | 17 | ||||
-rw-r--r-- | lib/api/entities.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/git/storage.rb | 25 | ||||
-rw-r--r-- | lib/gitlab/git/storage/checker.rb | 120 | ||||
-rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 78 | ||||
-rw-r--r-- | lib/gitlab/git/storage/circuit_breaker_settings.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/git/storage/failure_info.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 65 | ||||
-rw-r--r-- | lib/gitlab/git/storage/health.rb | 92 | ||||
-rw-r--r-- | lib/gitlab/git/storage/null_circuit_breaker.rb | 50 | ||||
-rw-r--r-- | lib/gitlab/storage_check.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/storage_check/cli.rb | 71 | ||||
-rw-r--r-- | lib/gitlab/storage_check/gitlab_caller.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/storage_check/option_parser.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/storage_check/response.rb | 77 |
16 files changed, 2 insertions, 768 deletions
diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index 6eddc5e5b61..da756daadcc 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -13,37 +13,24 @@ module API end resource ':type' do namespace '', requirements: { type: 'repository_storage' } do - helpers do - def failing_storage_health - @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages - end - - def storage_health - @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages - end - end - desc 'Get all git storages' do detail 'This feature was introduced in GitLab 9.5' - success Entities::RepositoryStorageHealth end get do - present storage_health, with: Entities::RepositoryStorageHealth + present [] end desc 'Get all failing git storages' do detail 'This feature was introduced in GitLab 9.5' - success Entities::RepositoryStorageHealth end get 'failing' do - present failing_storage_health, with: Entities::RepositoryStorageHealth + present [] end desc 'Reset all storage failures and open circuitbreaker' do detail 'This feature was introduced in GitLab 9.5' end delete do - Gitlab::Git::Storage::FailureInfo.reset_all! end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 120545792f2..5a4b85f98cf 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1364,12 +1364,6 @@ module API expose :submitted, as: :akismet_submitted end - class RepositoryStorageHealth < Grape::Entity - expose :storage_name - expose :failing_on_hosts - expose :total_failures - end - class CustomAttribute < Grape::Entity expose :key expose :value diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7732049b69b..4218e812146 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -96,10 +96,6 @@ module Gitlab raise Gitlab::Git::CommandError.new(e.message) end - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) - end - def exists? gitaly_repository_client.exists? end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb deleted file mode 100644 index 5933312b0b5..00000000000 --- a/lib/gitlab/git/storage.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module Git - module Storage - class Inaccessible < StandardError - attr_reader :retry_after - - def initialize(message = nil, retry_after = nil) - super(message) - @retry_after = retry_after - end - end - - CircuitOpen = Class.new(Inaccessible) - Misconfiguration = Class.new(Inaccessible) - Failing = Class.new(Inaccessible) - - REDIS_KEY_PREFIX = 'storage_accessible:'.freeze - REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze - - def self.redis - Gitlab::Redis::SharedState - end - end - end -end diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb deleted file mode 100644 index 391f0d70583..00000000000 --- a/lib/gitlab/git/storage/checker.rb +++ /dev/null @@ -1,120 +0,0 @@ -module Gitlab - module Git - module Storage - class Checker - include CircuitBreakerSettings - - attr_reader :storage_path, :storage, :hostname, :logger - METRICS_MUTEX = Mutex.new - STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze - - def self.check_all(logger = Rails.logger) - threads = Gitlab.config.repositories.storages.keys.map do |storage_name| - Thread.new do - Thread.current[:result] = new(storage_name, logger).check_with_lease - end - end - - threads.map do |thread| - thread.join - thread[:result] - end - end - - def self.check_histogram - @check_histogram ||= - METRICS_MUTEX.synchronize do - @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds, - 'Storage check time in seconds', - {}, - STORAGE_TIMING_BUCKETS - ) - end - end - - def initialize(storage, logger = Rails.logger) - @storage = storage - config = Gitlab.config.repositories.storages[@storage] - @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path } - @logger = logger - - @hostname = Gitlab::Environment.hostname - end - - def check_with_lease - lease_key = "storage_check:#{cache_key}" - lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) - result = { storage: storage, success: nil } - - if uuid = lease.try_obtain - result[:success] = check - - Gitlab::ExclusiveLease.cancel(lease_key, uuid) - else - logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") - end - - result - end - - def check - if perform_access_check - track_storage_accessible - true - else - track_storage_inaccessible - logger.error("#{hostname}: #{storage}: Not accessible.") - false - end - end - - private - - def perform_access_check - start_time = Gitlab::Metrics::System.monotonic_time - - Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) - ensure - execution_time = Gitlab::Metrics::System.monotonic_time - start_time - self.class.check_histogram.observe({ storage: storage }, execution_time) - end - - def track_storage_inaccessible - first_failure = current_failure_info.first_failure || Time.now - last_failure = Time.now - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, first_failure.to_i) - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - maintain_known_keys(redis) - end - end - end - - def track_storage_accessible - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, nil) - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - maintain_known_keys(redis) - end - end - end - - def maintain_known_keys(redis) - expire_time = Time.now.to_i + failure_reset_time - redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) - redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) - end - - def current_failure_info - FailureInfo.load(cache_key) - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb deleted file mode 100644 index fcee9ae566c..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Gitlab - module Git - module Storage - class CircuitBreaker - include CircuitBreakerSettings - - attr_reader :storage, - :hostname - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def self.for_storage(storage) - cached_circuitbreakers = Gitlab::SafeRequestStore.fetch(:circuitbreaker_cache) do - Hash.new do |hash, storage_name| - hash[storage_name] = build(storage_name) - end - end - - cached_circuitbreakers[storage] - end - - def self.build(storage, hostname = Gitlab::Environment.hostname) - config = Gitlab.config.repositories.storages[storage] - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - if !config.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config.legacy_disk_path.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) - else - new(storage, hostname) - end - end - end - - def initialize(storage, hostname) - @storage = storage - @hostname = hostname - end - - def perform - return yield unless enabled? - - check_storage_accessible! - - yield - end - - def circuit_broken? - return false if no_failures? - - failure_count > failure_count_threshold - end - - private - - # The circuitbreaker can be enabled for the entire fleet using a Feature - # flag. - # - # Enabling it for a single host can be done setting the - # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. - def enabled? - ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') - end - - def failure_info - @failure_info ||= FailureInfo.load(cache_key) - end - - def check_storage_accessible! - if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) - end - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb deleted file mode 100644 index c9e225f187d..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module Git - module Storage - module CircuitBreakerSettings - def failure_count_threshold - application_settings.circuitbreaker_failure_count_threshold - end - - def failure_reset_time - application_settings.circuitbreaker_failure_reset_time - end - - def storage_timeout - application_settings.circuitbreaker_storage_timeout - end - - def access_retries - application_settings.circuitbreaker_access_retries - end - - def check_interval - application_settings.circuitbreaker_check_interval - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - - def application_settings - Gitlab::CurrentSettings.current_application_settings - end - end - end - end -end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb deleted file mode 100644 index 1d28a850049..00000000000 --- a/lib/gitlab/git/storage/failure_info.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module Git - module Storage - class FailureInfo - attr_accessor :first_failure, :last_failure, :failure_count - - def self.reset_all! - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end - - Gitlab::SafeRequestStore.delete(:circuitbreaker_cache) - end - - def self.load(cache_key) - first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - first_failure = Time.at(first_failure.to_i) if first_failure.present? - - new(first_failure, last_failure, failure_count.to_i) - end - - def initialize(first_failure, last_failure, failure_count) - @first_failure = first_failure - @last_failure = last_failure - @failure_count = failure_count - end - - def no_failures? - first_failure.blank? && last_failure.blank? && failure_count == 0 - end - end - end - end -end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb deleted file mode 100644 index 0a4e557b59b..00000000000 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ /dev/null @@ -1,65 +0,0 @@ -module Gitlab - module Git - module Storage - module ForkedStorageCheck - extend self - - def storage_available?(path, timeout_seconds = 5, retries = 1) - partial_timeout = timeout_seconds / retries - status = timeout_check(path, partial_timeout) - - # If the status check did not succeed the first time, we retry a few - # more times to avoid one-off failures - current_attempts = 1 - while current_attempts < retries && !status.success? - status = timeout_check(path, partial_timeout) - current_attempts += 1 - end - - status.success? - end - - def timeout_check(path, timeout_seconds) - filesystem_check_pid = check_filesystem_in_process(path) - - deadline = timeout_seconds.seconds.from_now.utc - wait_time = 0.01 - status = nil - - while status.nil? - - if deadline > Time.now.utc - sleep(wait_time) - _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) - else - Process.kill('KILL', filesystem_check_pid) - # Blocking wait, so we are sure the process is gone before continuing - _pid, status = Process.wait2(filesystem_check_pid) - end - end - - status - end - - # This will spawn a new 2 processes to do the check: - # The outer child (waiter) will spawn another child process (stater). - # - # The stater is the process is performing the actual filesystem check - # the check might hang if the filesystem is acting up. - # In this case we will send a `KILL` to the waiter, which will still - # be responsive while the stater is hanging. - def check_filesystem_in_process(path) - spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null') - end - - def ruby_check - <<~RUBY_FILESYSTEM_CHECK - inner_pid = fork { File.stat(ARGV.first) } - Process.waitpid(inner_pid) - exit $?.exitstatus - RUBY_FILESYSTEM_CHECK - end - end - end - end -end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb deleted file mode 100644 index 8e14acb4ccb..00000000000 --- a/lib/gitlab/git/storage/health.rb +++ /dev/null @@ -1,92 +0,0 @@ -module Gitlab - module Git - module Storage - class Health - attr_reader :storage_name, :info - - def self.prefix_for_storage(storage_name) - "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:" - end - - def self.for_all_storages - storage_names = Gitlab.config.repositories.storages.keys - results_per_storage = nil - - Gitlab::Git::Storage.redis.with do |redis| - keys_per_storage = all_keys_for_storages(storage_names, redis) - results_per_storage = load_for_keys(keys_per_storage, redis) - end - - results_per_storage.map do |name, info| - info.each { |i| i[:failure_count] = i[:failure_count].value.to_i } - new(name, info) - end - end - - private_class_method def self.all_keys_for_storages(storage_names, redis) - keys_per_storage = {} - all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - - storage_names.each do |storage_name| - prefix = prefix_for_storage(storage_name) - - keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) } - end - - keys_per_storage - end - - private_class_method def self.load_for_keys(keys_per_storage, redis) - info_for_keys = {} - - redis.pipelined do - keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.map do |key| - { name: key, failure_count: redis.hget(key, :failure_count) } - end - - info_for_keys[storage_name] = info_for_storage - end - end - - info_for_keys - end - - def self.for_failing_storages - for_all_storages.select(&:failing?) - end - - def initialize(storage_name, info) - @storage_name = storage_name - @info = info - end - - def failing_info - @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 } - end - - def failing? - failing_info.any? - end - - def failing_on_hosts - @failing_on_hosts ||= failing_info.map do |info_for_host| - info_for_host[:name].split(':').last - end - end - - def failing_circuit_breakers - @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.build(storage_name, hostname) - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def total_failures - @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end -end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb deleted file mode 100644 index 261c936c689..00000000000 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Git - module Storage - class NullCircuitBreaker - include CircuitBreakerSettings - - # These will have actual values - attr_reader :storage, - :hostname - - # These will always have nil values - attr_reader :storage_path - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def initialize(storage, hostname, error: nil) - @storage = storage - @hostname = hostname - @error = error - end - - def perform - @error ? raise(@error) : yield - end - - def circuit_broken? - !!@error - end - - def backing_off? - false - end - - def failure_info - @failure_info ||= - if circuit_broken? - Gitlab::Git::Storage::FailureInfo.new(Time.now, - Time.now, - failure_count_threshold) - else - Gitlab::Git::Storage::FailureInfo.new(nil, - nil, - 0) - end - end - end - end - end -end diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb deleted file mode 100644 index fe81513c9ec..00000000000 --- a/lib/gitlab/storage_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -require_relative 'storage_check/cli' -require_relative 'storage_check/gitlab_caller' -require_relative 'storage_check/option_parser' -require_relative 'storage_check/response' - -module Gitlab - module StorageCheck - ENDPOINT = '/-/storage_check'.freeze - Options = Struct.new(:target, :token, :interval, :dryrun) - end -end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb deleted file mode 100644 index 9b64c8e033a..00000000000 --- a/lib/gitlab/storage_check/cli.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Gitlab - module StorageCheck - class CLI - def self.start!(args) - runner = new(Gitlab::StorageCheck::OptionParser.parse!(args)) - runner.start_loop - end - - attr_reader :logger, :options - - def initialize(options) - @options = options - @logger = Logger.new(STDOUT) - end - - def start_loop - logger.info "Checking #{options.target} every #{options.interval} seconds" - - if options.dryrun - logger.info "Dryrun, exiting..." - return - end - - begin - loop do - response = GitlabCaller.new(options).call! - log_response(response) - update_settings(response) - - sleep options.interval - end - rescue Interrupt - logger.info "Ending storage-check" - end - end - - def update_settings(response) - previous_interval = options.interval - - if response.valid? - options.interval = response.check_interval || previous_interval - end - - if previous_interval != options.interval - logger.info "Interval changed: #{options.interval} seconds" - end - end - - def log_response(response) - unless response.valid? - return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}") - end - - if response.responsive_shards.any? - logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}") - end - - warnings = [] - if response.skipped_shards.any? - warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" - end - - if response.failing_shards.any? - warnings << "Failing shards: #{response.failing_shards.join(', ')}" - end - - logger.warn(warnings.join(' - ')) if warnings.any? - end - end - end -end diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb deleted file mode 100644 index 44952b68844..00000000000 --- a/lib/gitlab/storage_check/gitlab_caller.rb +++ /dev/null @@ -1,39 +0,0 @@ -require 'excon' - -module Gitlab - module StorageCheck - class GitlabCaller - def initialize(options) - @options = options - end - - def call! - Gitlab::StorageCheck::Response.new(get_response) - rescue Errno::ECONNREFUSED, Excon::Error - # Server not ready, treated as invalid response. - Gitlab::StorageCheck::Response.new(nil) - end - - def get_response - scheme, *other_parts = URI.split(@options.target) - socket_path = if scheme == 'unix' - other_parts.compact.join - end - - connection = Excon.new(@options.target, socket: socket_path) - connection.post(path: Gitlab::StorageCheck::ENDPOINT, - headers: headers) - end - - def headers - @headers ||= begin - headers = {} - headers['Content-Type'] = headers['Accept'] = 'application/json' - headers['TOKEN'] = @options.token if @options.token - - headers - end - end - end - end -end diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb deleted file mode 100644 index 66ed7906f97..00000000000 --- a/lib/gitlab/storage_check/option_parser.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module StorageCheck - class OptionParser - def self.parse!(args) - # Start out with some defaults - options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false) - - parser = ::OptionParser.new do |opts| - opts.banner = "Usage: bin/storage_check [options]" - - opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value| - options.target = value - end - - opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value } - - opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value| - options.interval = value - end - - opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value| - options.dryrun = value - end - end - parser.parse!(args) - - unless options.target - raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks') - end - - if URI.parse(options.target).scheme.nil? - raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported') - end - - options - end - end - end -end diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb deleted file mode 100644 index 326ab236e3e..00000000000 --- a/lib/gitlab/storage_check/response.rb +++ /dev/null @@ -1,77 +0,0 @@ -require 'json' - -module Gitlab - module StorageCheck - class Response - attr_reader :http_response - - def initialize(http_response) - @http_response = http_response - end - - def valid? - @http_response && (200...299).cover?(@http_response.status) && - @http_response.headers['Content-Type'].include?('application/json') && - parsed_response - end - - def check_interval - return nil unless parsed_response - - parsed_response['check_interval'] - end - - def responsive_shards - divided_results[:responsive_shards] - end - - def skipped_shards - divided_results[:skipped_shards] - end - - def failing_shards - divided_results[:failing_shards] - end - - private - - def results - return [] unless parsed_response - - parsed_response['results'] - end - - def divided_results - return @divided_results if @divided_results - - @divided_results = {} - @divided_results[:responsive_shards] = [] - @divided_results[:skipped_shards] = [] - @divided_results[:failing_shards] = [] - - results.each do |info| - name = info['storage'] - - case info['success'] - when true - @divided_results[:responsive_shards] << name - when false - @divided_results[:failing_shards] << name - else - @divided_results[:skipped_shards] << name - end - end - - @divided_results - end - - def parsed_response - return @parsed_response if defined?(@parsed_response) - - @parsed_response = JSON.parse(@http_response.body) - rescue JSON::JSONError - @parsed_response = nil - end - end - end -end |