From 5ba9427f34800591cb45eb34403c71e2e65fd122 Mon Sep 17 00:00:00 2001 From: Lamont Granquist Date: Thu, 23 Apr 2015 17:21:31 -0700 Subject: change :validate_utf8=false to still emit utf8 validate_utf8=true raises on bad input validate_utf8=false now does not raise, but still produces valid UTF-8 clean JSON by replacing characters. the previous behavior of emitting bad JSON that was not UTF-8 clean when validate_utf8 was false is dropped (and this was behavior only of the C extension and not the ffi extension since the ffi layer must do some scrubbing of its own). --- lib/ffi_yajl/encoder.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/ffi_yajl/encoder.rb b/lib/ffi_yajl/encoder.rb index a89bcd3..8c6dce6 100644 --- a/lib/ffi_yajl/encoder.rb +++ b/lib/ffi_yajl/encoder.rb @@ -40,7 +40,8 @@ module FFI_Yajl # call either the ext or ffi hook str = do_yajl_encode(obj, yajl_gen_opts, opts) - str.force_encoding('UTF-8') if defined? Encoding + # we can skip cleaning the whole string for utf-8 issues if we have yajl validate as we go + str.encode("utf-8", "binary", :undef => :replace) unless yajl_gen_opts[:yajl_gen_validate_utf8] str end @@ -55,7 +56,7 @@ module FFI_Yajl def self.raise_error_for_status(status, token=nil) # scrub token to valid utf-8 since we may be issuing an exception on an invalid utf-8 token - token.encode!("utf-8", "binary", :undef => :replace) + token = token.to_s.encode("utf-8", "binary", :undef => :replace) case status when 1 # yajl_gen_keys_must_be_strings raise FFI_Yajl::EncodeError, "YAJL internal error: attempted use of non-string object as key" -- cgit v1.2.1