Exception encoding

Raise with messages in UTF-8 encoding.

4 files changed, 139 insertions, 83 deletions

diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb
index ae4f861..2addc53 100644
--- a/ext/json/ext/parser/extconf.rb
+++ b/ext/json/ext/parser/extconf.rb
@@ -1,3 +1,5 @@
 require 'mkmf'
 
+have_func("rb_enc_raise", "ruby.h")
+
 create_makefile 'json/ext/parser'
diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c
index 4bbd30c..94543d2 100644
--- a/ext/json/ext/parser/parser.c
+++ b/ext/json/ext/parser/parser.c
@@ -3,6 +3,28 @@
 #include "../fbuffer/fbuffer.h"
 #include "parser.h"
 
+#if defined HAVE_RUBY_ENCODING_H
+# define EXC_ENCODING rb_utf8_encoding(),
+# ifndef HAVE_RB_ENC_RAISE
+static void
+enc_raise(rb_encoding *enc, VALUE exc, const char *fmt, ...)
+{
+    va_list args;
+    VALUE mesg;
+
+    va_start(args, fmt);
+    mesg = rb_enc_vsprintf(enc, fmt, args);
+    va_end(args);
+
+    rb_exc_raise(rb_exc_new3(exc, mesg));
+}
+#   define rb_enc_raise enc_raise
+# endif
+#else
+# define EXC_ENCODING /* nothing */
+# define rb_enc_raise rb_raise
+#endif
+
 /* unicode */
 
 static const char digit_values[256] = {
@@ -76,11 +98,11 @@ static ID i_json_creatable_p, i_json_create, i_create_id, i_create_additions,
           i_match_string, i_aset, i_aref, i_leftshift;
 
 
-#line 102 "parser.rl"
+#line 124 "parser.rl"
 
 
 
-#line 84 "parser.c"
+#line 106 "parser.c"
 enum {JSON_object_start = 1};
 enum {JSON_object_first_final = 27};
 enum {JSON_object_error = 0};
@@ -88,7 +110,7 @@ enum {JSON_object_error = 0};
 enum {JSON_object_en_main = 1};
 
 
-#line 143 "parser.rl"
+#line 165 "parser.rl"
 
 
 static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *result)
@@ -104,14 +126,14 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
     *result = NIL_P(object_class) ? rb_hash_new() : rb_class_new_instance(0, 0, object_class);
 
 
-#line 108 "parser.c"
+#line 130 "parser.c"
 	{
 	cs = JSON_object_start;
 	}
 
-#line 158 "parser.rl"
+#line 180 "parser.rl"
 
-#line 115 "parser.c"
+#line 137 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -139,7 +161,7 @@ case 2:
 		goto st2;
 	goto st0;
 tr2:
-#line 125 "parser.rl"
+#line 147 "parser.rl"
 	{
         char *np;
         json->parsing_name = 1;
@@ -152,7 +174,7 @@ st3:
 	if ( ++p == pe )
 		goto _test_eof3;
 case 3:
-#line 156 "parser.c"
+#line 178 "parser.c"
 	switch( (*p) ) {
 		case 13: goto st3;
 		case 32: goto st3;
@@ -219,7 +241,7 @@ case 8:
 		goto st8;
 	goto st0;
 tr11:
-#line 110 "parser.rl"
+#line 132 "parser.rl"
 	{
         VALUE v = Qnil;
         char *np = JSON_parse_value(json, p, pe, &v);
@@ -239,7 +261,7 @@ st9:
 	if ( ++p == pe )
 		goto _test_eof9;
 case 9:
-#line 243 "parser.c"
+#line 265 "parser.c"
 	switch( (*p) ) {
 		case 13: goto st9;
 		case 32: goto st9;
@@ -328,14 +350,14 @@ case 18:
 		goto st9;
 	goto st18;
 tr4:
-#line 133 "parser.rl"
+#line 155 "parser.rl"
 	{ p--; {p++; cs = 27; goto _out;} }
 	goto st27;
 st27:
 	if ( ++p == pe )
 		goto _test_eof27;
 case 27:
-#line 339 "parser.c"
+#line 361 "parser.c"
 	goto st0;
 st19:
 	if ( ++p == pe )
@@ -433,7 +455,7 @@ case 26:
 	_out: {}
 	}
 
-#line 159 "parser.rl"
+#line 181 "parser.rl"
 
     if (cs >= JSON_object_first_final) {
         if (json->create_additions) {
@@ -458,7 +480,7 @@ case 26:
 
 
 
-#line 462 "parser.c"
+#line 484 "parser.c"
 enum {JSON_value_start = 1};
 enum {JSON_value_first_final = 29};
 enum {JSON_value_error = 0};
@@ -466,7 +488,7 @@ enum {JSON_value_error = 0};
 enum {JSON_value_en_main = 1};
 
 
-#line 263 "parser.rl"
+#line 285 "parser.rl"
 
 
 static char *JSON_parse_value(JSON_Parser *json, char *p, char *pe, VALUE *result)
@@ -474,14 +496,14 @@ static char *JSON_parse_value(JSON_Parser *json, char *p, char *pe, VALUE *resul
     int cs = EVIL;
 
 
-#line 478 "parser.c"
+#line 500 "parser.c"
 	{
 	cs = JSON_value_start;
 	}
 
-#line 270 "parser.rl"
+#line 292 "parser.rl"
 
-#line 485 "parser.c"
+#line 507 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -515,14 +537,14 @@ st0:
 cs = 0;
 	goto _out;
 tr2:
-#line 211 "parser.rl"
+#line 233 "parser.rl"
 	{
         char *np = JSON_parse_string(json, p, pe, result);
         if (np == NULL) { p--; {p++; cs = 29; goto _out;} } else {p = (( np))-1;}
     }
 	goto st29;
 tr3:
-#line 216 "parser.rl"
+#line 238 "parser.rl"
 	{
         char *np;
         if(pe > p + 8 && !strncmp(MinusInfinity, p, 9)) {
@@ -531,7 +553,7 @@ tr3:
                 {p = (( p + 10))-1;}
                 p--; {p++; cs = 29; goto _out;}
             } else {
-                rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+                rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
             }
         }
         np = JSON_parse_float(json, p, pe, result);
@@ -542,7 +564,7 @@ tr3:
     }
 	goto st29;
 tr7:
-#line 234 "parser.rl"
+#line 256 "parser.rl"
 	{
         char *np;
         json->current_nesting++;
@@ -552,7 +574,7 @@ tr7:
     }
 	goto st29;
 tr11:
-#line 242 "parser.rl"
+#line 264 "parser.rl"
 	{
         char *np;
         json->current_nesting++;
@@ -562,39 +584,39 @@ tr11:
     }
 	goto st29;
 tr25:
-#line 204 "parser.rl"
+#line 226 "parser.rl"
 	{
         if (json->allow_nan) {
             *result = CInfinity;
         } else {
-            rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p - 8);
+            rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p - 8);
         }
     }
 	goto st29;
 tr27:
-#line 197 "parser.rl"
+#line 219 "parser.rl"
 	{
         if (json->allow_nan) {
             *result = CNaN;
         } else {
-            rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p - 2);
+            rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p - 2);
         }
     }
 	goto st29;
 tr31:
-#line 191 "parser.rl"
+#line 213 "parser.rl"
 	{
         *result = Qfalse;
     }
 	goto st29;
 tr34:
-#line 188 "parser.rl"
+#line 210 "parser.rl"
 	{
         *result = Qnil;
     }
 	goto st29;
 tr37:
-#line 194 "parser.rl"
+#line 216 "parser.rl"
 	{
         *result = Qtrue;
     }
@@ -603,9 +625,9 @@ st29:
 	if ( ++p == pe )
 		goto _test_eof29;
 case 29:
-#line 250 "parser.rl"
+#line 272 "parser.rl"
 	{ p--; {p++; cs = 29; goto _out;} }
-#line 609 "parser.c"
+#line 631 "parser.c"
 	switch( (*p) ) {
 		case 13: goto st29;
 		case 32: goto st29;
@@ -846,7 +868,7 @@ case 28:
 	_out: {}
 	}
 
-#line 271 "parser.rl"
+#line 293 "parser.rl"
 
     if (cs >= JSON_value_first_final) {
         return p;
@@ -856,7 +878,7 @@ case 28:
 }
 
 
-#line 860 "parser.c"
+#line 882 "parser.c"
 enum {JSON_integer_start = 1};
 enum {JSON_integer_first_final = 3};
 enum {JSON_integer_error = 0};
@@ -864,7 +886,7 @@ enum {JSON_integer_error = 0};
 enum {JSON_integer_en_main = 1};
 
 
-#line 287 "parser.rl"
+#line 309 "parser.rl"
 
 
 static char *JSON_parse_integer(JSON_Parser *json, char *p, char *pe, VALUE *result)
@@ -872,15 +894,15 @@ static char *JSON_parse_integer(JSON_Parser *json, char *p, char *pe, VALUE *res
     int cs = EVIL;
 
 
-#line 876 "parser.c"
+#line 898 "parser.c"
 	{
 	cs = JSON_integer_start;
 	}
 
-#line 294 "parser.rl"
+#line 316 "parser.rl"
     json->memo = p;
 
-#line 884 "parser.c"
+#line 906 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -914,14 +936,14 @@ case 3:
 		goto st0;
 	goto tr4;
 tr4:
-#line 284 "parser.rl"
+#line 306 "parser.rl"
 	{ p--; {p++; cs = 4; goto _out;} }
 	goto st4;
 st4:
 	if ( ++p == pe )
 		goto _test_eof4;
 case 4:
-#line 925 "parser.c"
+#line 947 "parser.c"
 	goto st0;
 st5:
 	if ( ++p == pe )
@@ -940,7 +962,7 @@ case 5:
 	_out: {}
 	}
 
-#line 296 "parser.rl"
+#line 318 "parser.rl"
 
     if (cs >= JSON_integer_first_final) {
         long len = p - json->memo;
@@ -955,7 +977,7 @@ case 5:
 }
 
 
-#line 959 "parser.c"
+#line 981 "parser.c"
 enum {JSON_float_start = 1};
 enum {JSON_float_first_final = 8};
 enum {JSON_float_error = 0};
@@ -963,7 +985,7 @@ enum {JSON_float_error = 0};
 enum {JSON_float_en_main = 1};
 
 
-#line 321 "parser.rl"
+#line 343 "parser.rl"
 
 
 static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *result)
@@ -971,15 +993,15 @@ static char *JSON_parse_float(JSON_Parser *json, char *p, char *pe, VALUE *resul
     int cs = EVIL;
 
 
-#line 975 "parser.c"
+#line 997 "parser.c"
 	{
 	cs = JSON_float_start;
 	}
 
-#line 328 "parser.rl"
+#line 350 "parser.rl"
     json->memo = p;
 
-#line 983 "parser.c"
+#line 1005 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -1037,14 +1059,14 @@ case 8:
 		goto st0;
 	goto tr9;
 tr9:
-#line 315 "parser.rl"
+#line 337 "parser.rl"
 	{ p--; {p++; cs = 9; goto _out;} }
 	goto st9;
 st9:
 	if ( ++p == pe )
 		goto _test_eof9;
 case 9:
-#line 1048 "parser.c"
+#line 1070 "parser.c"
 	goto st0;
 st5:
 	if ( ++p == pe )
@@ -1105,7 +1127,7 @@ case 7:
 	_out: {}
 	}
 
-#line 330 "parser.rl"
+#line 352 "parser.rl"
 
     if (cs >= JSON_float_first_final) {
         long len = p - json->memo;
@@ -1121,7 +1143,7 @@ case 7:
 
 
 
-#line 1125 "parser.c"
+#line 1147 "parser.c"
 enum {JSON_array_start = 1};
 enum {JSON_array_first_final = 17};
 enum {JSON_array_error = 0};
@@ -1129,7 +1151,7 @@ enum {JSON_array_error = 0};
 enum {JSON_array_en_main = 1};
 
 
-#line 373 "parser.rl"
+#line 395 "parser.rl"
 
 
 static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *result)
@@ -1143,14 +1165,14 @@ static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *resul
     *result = NIL_P(array_class) ? rb_ary_new() : rb_class_new_instance(0, 0, array_class);
 
 
-#line 1147 "parser.c"
+#line 1169 "parser.c"
 	{
 	cs = JSON_array_start;
 	}
 
-#line 386 "parser.rl"
+#line 408 "parser.rl"
 
-#line 1154 "parser.c"
+#line 1176 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -1189,7 +1211,7 @@ case 2:
 		goto st2;
 	goto st0;
 tr2:
-#line 350 "parser.rl"
+#line 372 "parser.rl"
 	{
         VALUE v = Qnil;
         char *np = JSON_parse_value(json, p, pe, &v);
@@ -1209,7 +1231,7 @@ st3:
 	if ( ++p == pe )
 		goto _test_eof3;
 case 3:
-#line 1213 "parser.c"
+#line 1235 "parser.c"
 	switch( (*p) ) {
 		case 13: goto st3;
 		case 32: goto st3;
@@ -1309,14 +1331,14 @@ case 12:
 		goto st3;
 	goto st12;
 tr4:
-#line 365 "parser.rl"
+#line 387 "parser.rl"
 	{ p--; {p++; cs = 17; goto _out;} }
 	goto st17;
 st17:
 	if ( ++p == pe )
 		goto _test_eof17;
 case 17:
-#line 1320 "parser.c"
+#line 1342 "parser.c"
 	goto st0;
 st13:
 	if ( ++p == pe )
@@ -1372,12 +1394,12 @@ case 16:
 	_out: {}
 	}
 
-#line 387 "parser.rl"
+#line 409 "parser.rl"
 
     if(cs >= JSON_array_first_final) {
         return p + 1;
     } else {
-        rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+        rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
         return NULL;
     }
 }
@@ -1453,7 +1475,7 @@ static VALUE json_string_unescape(VALUE result, char *string, char *stringEnd)
 }
 
 
-#line 1457 "parser.c"
+#line 1479 "parser.c"
 enum {JSON_string_start = 1};
 enum {JSON_string_first_final = 8};
 enum {JSON_string_error = 0};
@@ -1461,7 +1483,7 @@ enum {JSON_string_error = 0};
 enum {JSON_string_en_main = 1};
 
 
-#line 486 "parser.rl"
+#line 508 "parser.rl"
 
 
 static int
@@ -1483,15 +1505,15 @@ static char *JSON_parse_string(JSON_Parser *json, char *p, char *pe, VALUE *resu
 
     *result = rb_str_buf_new(0);
 
-#line 1487 "parser.c"
+#line 1509 "parser.c"
 	{
 	cs = JSON_string_start;
 	}
 
-#line 507 "parser.rl"
+#line 529 "parser.rl"
     json->memo = p;
 
-#line 1495 "parser.c"
+#line 1517 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -1516,7 +1538,7 @@ case 2:
 		goto st0;
 	goto st2;
 tr2:
-#line 472 "parser.rl"
+#line 494 "parser.rl"
 	{
         *result = json_string_unescape(*result, json->memo + 1, p);
         if (NIL_P(*result)) {
@@ -1527,14 +1549,14 @@ tr2:
             {p = (( p + 1))-1;}
         }
     }
-#line 483 "parser.rl"
+#line 505 "parser.rl"
 	{ p--; {p++; cs = 8; goto _out;} }
 	goto st8;
 st8:
 	if ( ++p == pe )
 		goto _test_eof8;
 case 8:
-#line 1538 "parser.c"
+#line 1560 "parser.c"
 	goto st0;
 st3:
 	if ( ++p == pe )
@@ -1610,7 +1632,7 @@ case 7:
 	_out: {}
 	}
 
-#line 509 "parser.rl"
+#line 531 "parser.rl"
 
     if (json->create_additions && RTEST(match_string = json->match_string)) {
           VALUE klass;
@@ -1775,7 +1797,7 @@ static VALUE cParser_initialize(int argc, VALUE *argv, VALUE self)
 }
 
 
-#line 1779 "parser.c"
+#line 1801 "parser.c"
 enum {JSON_start = 1};
 enum {JSON_first_final = 10};
 enum {JSON_error = 0};
@@ -1783,7 +1805,7 @@ enum {JSON_error = 0};
 enum {JSON_en_main = 1};
 
 
-#line 687 "parser.rl"
+#line 709 "parser.rl"
 
 
 /*
@@ -1800,16 +1822,16 @@ static VALUE cParser_parse(VALUE self)
   GET_PARSER;
 
 
-#line 1804 "parser.c"
+#line 1826 "parser.c"
 	{
 	cs = JSON_start;
 	}
 
-#line 703 "parser.rl"
+#line 725 "parser.rl"
   p = json->source;
   pe = p + json->len;
 
-#line 1813 "parser.c"
+#line 1835 "parser.c"
 	{
 	if ( p == pe )
 		goto _test_eof;
@@ -1843,7 +1865,7 @@ st0:
 cs = 0;
 	goto _out;
 tr2:
-#line 679 "parser.rl"
+#line 701 "parser.rl"
 	{
         char *np = JSON_parse_value(json, p, pe, &result);
         if (np == NULL) { p--; {p++; cs = 10; goto _out;} } else {p = (( np))-1;}
@@ -1853,7 +1875,7 @@ st10:
 	if ( ++p == pe )
 		goto _test_eof10;
 case 10:
-#line 1857 "parser.c"
+#line 1879 "parser.c"
 	switch( (*p) ) {
 		case 13: goto st10;
 		case 32: goto st10;
@@ -1942,12 +1964,12 @@ case 9:
 	_out: {}
 	}
 
-#line 706 "parser.rl"
+#line 728 "parser.rl"
 
   if (cs >= JSON_first_final && p == pe) {
     return result;
   } else {
-    rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+    rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
     return Qnil;
   }
 }
diff --git a/ext/json/ext/parser/parser.rl b/ext/json/ext/parser/parser.rl
index f35336c..fa2880f 100644
--- a/ext/json/ext/parser/parser.rl
+++ b/ext/json/ext/parser/parser.rl
@@ -1,6 +1,28 @@
 #include "../fbuffer/fbuffer.h"
 #include "parser.h"
 
+#if defined HAVE_RUBY_ENCODING_H
+# define EXC_ENCODING rb_utf8_encoding(),
+# ifndef HAVE_RB_ENC_RAISE
+static void
+enc_raise(rb_encoding *enc, VALUE exc, const char *fmt, ...)
+{
+    va_list args;
+    VALUE mesg;
+
+    va_start(args, fmt);
+    mesg = rb_enc_vsprintf(enc, fmt, args);
+    va_end(args);
+
+    rb_exc_raise(rb_exc_new3(exc, mesg));
+}
+#   define rb_enc_raise enc_raise
+# endif
+#else
+# define EXC_ENCODING /* nothing */
+# define rb_enc_raise rb_raise
+#endif
+
 /* unicode */
 
 static const char digit_values[256] = {
@@ -198,14 +220,14 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
         if (json->allow_nan) {
             *result = CNaN;
         } else {
-            rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p - 2);
+            rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p - 2);
         }
     }
     action parse_infinity {
         if (json->allow_nan) {
             *result = CInfinity;
         } else {
-            rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p - 8);
+            rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p - 8);
         }
     }
     action parse_string {
@@ -221,7 +243,7 @@ static char *JSON_parse_object(JSON_Parser *json, char *p, char *pe, VALUE *resu
                 fexec p + 10;
                 fhold; fbreak;
             } else {
-                rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+                rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
             }
         }
         np = JSON_parse_float(json, fpc, pe, result);
@@ -388,7 +410,7 @@ static char *JSON_parse_array(JSON_Parser *json, char *p, char *pe, VALUE *resul
     if(cs >= JSON_array_first_final) {
         return p + 1;
     } else {
-        rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+        rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
         return NULL;
     }
 }
@@ -707,7 +729,7 @@ static VALUE cParser_parse(VALUE self)
   if (cs >= JSON_first_final && p == pe) {
     return result;
   } else {
-    rb_raise(eParserError, "%u: unexpected token at '%s'", __LINE__, p);
+    rb_enc_raise(EXC_ENCODING eParserError, "%u: unexpected token at '%s'", __LINE__, p);
     return Qnil;
   }
 }
diff --git a/tests/json_parser_test.rb b/tests/json_parser_test.rb
index 46fa47c..ee28578 100644
--- a/tests/json_parser_test.rb
+++ b/tests/json_parser_test.rb
@@ -17,6 +17,16 @@ class JSONParserTest < Test::Unit::TestCase
     assert_equal Encoding::UTF_16, source.encoding
   end if defined?(Encoding::UTF_16)
 
+  def test_error_message_encoding
+    bug10705 = '[ruby-core:67386] [Bug #10705]'
+    json = ".\"\xE2\x88\x9A\"".force_encoding(Encoding::UTF_8)
+    e = assert_raise(JSON::ParserError) {
+      JSON::Ext::Parser.new(json).parse
+    }
+    assert_equal(Encoding::UTF_8, e.message.encoding, bug10705)
+    assert_include(e.message, json, bug10705)
+  end if defined?(Encoding::UTF_8) and defined?(JSON::Ext::Parser)
+
   def test_parsing
     parser = JSON::Parser.new('"test"')
     assert_equal 'test', parser.parse