From 5ba9427f34800591cb45eb34403c71e2e65fd122 Mon Sep 17 00:00:00 2001 From: Lamont Granquist Date: Thu, 23 Apr 2015 17:21:31 -0700 Subject: change :validate_utf8=false to still emit utf8 validate_utf8=true raises on bad input validate_utf8=false now does not raise, but still produces valid UTF-8 clean JSON by replacing characters. the previous behavior of emitting bad JSON that was not UTF-8 clean when validate_utf8 was false is dropped (and this was behavior only of the C extension and not the ffi extension since the ffi layer must do some scrubbing of its own). --- README.md | 8 +++++--- lib/ffi_yajl/encoder.rb | 5 +++-- spec/ffi_yajl/encoder_spec.rb | 4 ++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c6459ed..a1b7ca2 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ hash = parser.parse( json ) * `:check_utf8` * `:dont_validate_strings` * `:symbolize_keys` (default = false): -* `:symbolize_names` (default = false): alias for `:symbolize_keys` +* `:symbolize_names` (default = false): Alias for `:symbolize_keys`. * `:allow_trailing_garbage` * `:allow_multiple_values` * `:allow_partial_values` @@ -79,8 +79,10 @@ hash = parser.parse( json ) ## Encoder Options -* `:pretty` (default = false): -* `:validate_utf8` +* `:pretty` (default = false): Produces more human readable 'pretty' output. +* `:validate_utf8` (default = true): Will raise an exception when trying to + encode strings that are invalid UTF-8. When set to false this still will + procude valid UTF-8 JSON but will replace invalid characters. ## Forcing FFI or C Extension diff --git a/lib/ffi_yajl/encoder.rb b/lib/ffi_yajl/encoder.rb index a89bcd3..8c6dce6 100644 --- a/lib/ffi_yajl/encoder.rb +++ b/lib/ffi_yajl/encoder.rb @@ -40,7 +40,8 @@ module FFI_Yajl # call either the ext or ffi hook str = do_yajl_encode(obj, yajl_gen_opts, opts) - str.force_encoding('UTF-8') if defined? Encoding + # we can skip cleaning the whole string for utf-8 issues if we have yajl validate as we go + str.encode("utf-8", "binary", :undef => :replace) unless yajl_gen_opts[:yajl_gen_validate_utf8] str end @@ -55,7 +56,7 @@ module FFI_Yajl def self.raise_error_for_status(status, token=nil) # scrub token to valid utf-8 since we may be issuing an exception on an invalid utf-8 token - token.encode!("utf-8", "binary", :undef => :replace) + token = token.to_s.encode("utf-8", "binary", :undef => :replace) case status when 1 # yajl_gen_keys_must_be_strings raise FFI_Yajl::EncodeError, "YAJL internal error: attempted use of non-string object as key" diff --git a/spec/ffi_yajl/encoder_spec.rb b/spec/ffi_yajl/encoder_spec.rb index 5691bf6..7dbe4d0 100644 --- a/spec/ffi_yajl/encoder_spec.rb +++ b/spec/ffi_yajl/encoder_spec.rb @@ -196,6 +196,10 @@ describe "FFI_Yajl::Encoder" do it "does not raise an error" do expect{ encoder.encode(ruby) }.not_to raise_error end + + it "returns valid utf8" do + expect( encoder.encode(ruby).valid_encoding? ).to be true + end end end end -- cgit v1.2.1