diff options
author | Lamont Granquist <lamont@scriptkiddie.org> | 2015-04-23 17:21:31 -0700 |
---|---|---|
committer | Lamont Granquist <lamont@scriptkiddie.org> | 2015-04-23 17:21:31 -0700 |
commit | 5ba9427f34800591cb45eb34403c71e2e65fd122 (patch) | |
tree | 55b986afecd77faa62e0a2bb57b90fb3fc88c6b7 | |
parent | a1e388d59d98616d5d8ff79d7ae8a35030005ce8 (diff) | |
download | ffi-yajl-5ba9427f34800591cb45eb34403c71e2e65fd122.tar.gz |
change :validate_utf8=false to still emit utf8lcg/utf8-string-replacement
validate_utf8=true raises on bad input
validate_utf8=false now does not raise, but still produces valid
UTF-8 clean JSON by replacing characters.
the previous behavior of emitting bad JSON that was not UTF-8 clean
when validate_utf8 was false is dropped (and this was behavior only
of the C extension and not the ffi extension since the ffi layer must
do some scrubbing of its own).
-rw-r--r-- | README.md | 8 | ||||
-rw-r--r-- | lib/ffi_yajl/encoder.rb | 5 | ||||
-rw-r--r-- | spec/ffi_yajl/encoder_spec.rb | 4 |
3 files changed, 12 insertions, 5 deletions
@@ -71,7 +71,7 @@ hash = parser.parse( json ) * `:check_utf8` * `:dont_validate_strings` * `:symbolize_keys` (default = false): -* `:symbolize_names` (default = false): alias for `:symbolize_keys` +* `:symbolize_names` (default = false): Alias for `:symbolize_keys`. * `:allow_trailing_garbage` * `:allow_multiple_values` * `:allow_partial_values` @@ -79,8 +79,10 @@ hash = parser.parse( json ) ## Encoder Options -* `:pretty` (default = false): -* `:validate_utf8` +* `:pretty` (default = false): Produces more human readable 'pretty' output. +* `:validate_utf8` (default = true): Will raise an exception when trying to + encode strings that are invalid UTF-8. When set to false this still will + procude valid UTF-8 JSON but will replace invalid characters. ## Forcing FFI or C Extension diff --git a/lib/ffi_yajl/encoder.rb b/lib/ffi_yajl/encoder.rb index a89bcd3..8c6dce6 100644 --- a/lib/ffi_yajl/encoder.rb +++ b/lib/ffi_yajl/encoder.rb @@ -40,7 +40,8 @@ module FFI_Yajl # call either the ext or ffi hook str = do_yajl_encode(obj, yajl_gen_opts, opts) - str.force_encoding('UTF-8') if defined? Encoding + # we can skip cleaning the whole string for utf-8 issues if we have yajl validate as we go + str.encode("utf-8", "binary", :undef => :replace) unless yajl_gen_opts[:yajl_gen_validate_utf8] str end @@ -55,7 +56,7 @@ module FFI_Yajl def self.raise_error_for_status(status, token=nil) # scrub token to valid utf-8 since we may be issuing an exception on an invalid utf-8 token - token.encode!("utf-8", "binary", :undef => :replace) + token = token.to_s.encode("utf-8", "binary", :undef => :replace) case status when 1 # yajl_gen_keys_must_be_strings raise FFI_Yajl::EncodeError, "YAJL internal error: attempted use of non-string object as key" diff --git a/spec/ffi_yajl/encoder_spec.rb b/spec/ffi_yajl/encoder_spec.rb index 5691bf6..7dbe4d0 100644 --- a/spec/ffi_yajl/encoder_spec.rb +++ b/spec/ffi_yajl/encoder_spec.rb @@ -196,6 +196,10 @@ describe "FFI_Yajl::Encoder" do it "does not raise an error" do expect{ encoder.encode(ruby) }.not_to raise_error end + + it "returns valid utf8" do + expect( encoder.encode(ruby).valid_encoding? ).to be true + end end end end |