MDEV-26724 Endless loop in json_escape_to_string upon ... empty string

.. part#2: correctly pass the charset to JSON [un]escape functions

3 files changed, 50 insertions, 7 deletions

diff --git a/mysql-test/main/statistics_json.result b/mysql-test/main/statistics_json.result
index 77d4fecdd29..8a434e891fc 100644
--- a/mysql-test/main/statistics_json.result
+++ b/mysql-test/main/statistics_json.result
@@ -7898,6 +7898,34 @@ a
 
 b
 drop table t1;
+create table t1 (a char(1)) character set latin1;
+insert into t1 values (0xD1);
+select hex(a) from t1;
+hex(a)
+D1
+set histogram_type='json_hb';
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+select decode_histogram(hist_type, histogram)
+from mysql.column_stats
+where db_name=database() and table_name='t1';
+decode_histogram(hist_type, histogram)
+{
+  "histogram_hb_v2": [
+    {
+      "start": "Ñ",
+      "end": "Ñ",
+      "size": 1,
+      "ndv": 1
+    }
+  ]
+}
+select * from t1;
+a
+Ñ
+drop table t1;
 #
 # ASAN use-after-poison my_strnxfrm_simple_internal / Histogram_json_hb::range_selectivity ...
 # (Just the testcase)
diff --git a/mysql-test/main/statistics_json.test b/mysql-test/main/statistics_json.test
index fa581f54671..51ca9fe24f1 100644
--- a/mysql-test/main/statistics_json.test
+++ b/mysql-test/main/statistics_json.test
@@ -215,6 +215,19 @@ ANALYZE TABLE t PERSISTENT FOR ALL;
 select * from t1;
 drop table t1;
 
+create table t1 (a char(1)) character set latin1;
+insert into t1 values (0xD1);
+select hex(a) from t1;
+set histogram_type='json_hb';
+analyze table t1 persistent for all;
+
+select decode_histogram(hist_type, histogram)
+from mysql.column_stats
+where db_name=database() and table_name='t1';
+
+select * from t1;
+drop table t1;
+
 --echo #
 --echo # ASAN use-after-poison my_strnxfrm_simple_internal / Histogram_json_hb::range_selectivity ...
 --echo # (Just the testcase)
diff --git a/sql/opt_histogram_json.cc b/sql/opt_histogram_json.cc
index 385d5b1e4e6..7d270ba7191 100644
--- a/sql/opt_histogram_json.cc
+++ b/sql/opt_histogram_json.cc
@@ -39,7 +39,7 @@ static bool json_unescape_to_string(const char *val, int val_len, String* out)
     int res= json_unescape(&my_charset_utf8mb4_bin,
                            (const uchar*)val,
                            (const uchar*)val + val_len,
-                           &my_charset_utf8mb4_bin,
+                           out->charset(),
                            buf, buf + out->length());
     if (res >= 0)
     {
@@ -58,7 +58,7 @@ static bool json_unescape_to_string(const char *val, int val_len, String* out)
   Escape a JSON string and save it into *out.
 */
 
-static bool json_escape_to_string(const char *val, int val_len, String* out)
+static bool json_escape_to_string(const String *str, String* out)
 {
   // Make sure 'out' has some memory allocated.
   if (!out->alloced_length() && out->alloc(128))
@@ -68,10 +68,11 @@ static bool json_escape_to_string(const char *val, int val_len, String* out)
   {
     uchar *buf= (uchar*)out->ptr();
     out->length(out->alloced_length());
+    const uchar *str_ptr= (const uchar*)str->ptr();
 
-    int res= json_escape(&my_charset_utf8mb4_bin,
-                         (const uchar*)val,
-                         (const uchar*)val + val_len,
+    int res= json_escape(str->charset(),
+                         str_ptr,
+                         str_ptr + str->length(),
                          &my_charset_utf8mb4_bin,
                          buf, buf + out->length());
     if (res >= 0)
@@ -200,7 +201,7 @@ private:
 
     // Escape the value for JSON
     StringBuffer<MAX_FIELD_WIDTH> escaped_val;
-    if (json_escape_to_string(str->ptr(), str->length(), &escaped_val))
+    if (json_escape_to_string(str, &escaped_val))
       return true;
 
     // Note: The Json_writer does NOT do escapes (perhaps this should change?)
@@ -473,6 +474,7 @@ bool Histogram_json_hb::parse(MEM_ROOT *mem_root, Field *field,
       goto error;
     }
 
+    unescape_buf.set_charset(field->charset());
     uint len_to_copy= field->key_length();
     if (json_unescape_to_string(val, val_len, &unescape_buf))
     {
@@ -481,7 +483,7 @@ bool Histogram_json_hb::parse(MEM_ROOT *mem_root, Field *field,
       goto error;
     }
     field->store_text(unescape_buf.ptr(), unescape_buf.length(),
-                      &my_charset_bin);
+                      unescape_buf.charset());
     value_buf.alloc(field->pack_length());
     uint bytes= field->get_key_image((uchar*)value_buf.ptr(), len_to_copy,
                                      Field::itRAW);