MDEV-9811 LOAD DATA INFILE does not work well with gbk in some cases

MDEV-9824 LOAD DATA does not work with multi-byte strings in LINES TERMINATED BY when IGNORE is specified

9 files changed, 165 insertions, 21 deletions

diff --git a/include/m_ctype.h b/include/m_ctype.h
index d22a0b1f987..c892d576102 100644
--- a/include/m_ctype.h
+++ b/include/m_ctype.h
@@ -180,6 +180,10 @@ extern MY_UNI_CTYPE my_uni_ctype[256];
 /* A helper macros for "need at least n bytes" */
 #define MY_CS_TOOSMALLN(n)    (-100-(n))
 
+#define MY_CS_MBMAXLEN  6     /* Maximum supported mbmaxlen */
+#define MY_CS_IS_TOOSMALL(rc) ((rc) >= MY_CS_TOOSMALL6 && (rc) <= MY_CS_TOOSMALL)
+
+
 #define MY_SEQ_INTTAIL	1
 #define MY_SEQ_SPACES	2
 
diff --git a/mysql-test/r/ctype_gbk.result b/mysql-test/r/ctype_gbk.result
index b5774548d85..e454347592c 100644
--- a/mysql-test/r/ctype_gbk.result
+++ b/mysql-test/r/ctype_gbk.result
@@ -5926,3 +5926,24 @@ Warning	1300	Invalid gb2312 character string: '\xA3A'
 #
 # End of 10.1 tests
 #
+#
+# Start of 10.2 tests
+#
+#
+# MDEV-9811 LOAD DATA INFILE does not work well with gbk in some cases
+#
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET gbk);
+LOAD DATA INFILE '../../std_data/loaddata/mdev8711.txt' INTO TABLE t1 CHARACTER SET gbk LINES TERMINATED BY '@';
+SELECT HEX(a) FROM t1;
+HEX(a)
+B04061B041
+B042
+DELETE FROM t1;
+LOAD DATA INFILE '../../std_data/loaddata/mdev8711.txt' INTO TABLE t1 CHARACTER SET gbk LINES TERMINATED BY '@' IGNORE 1 LINES;
+SELECT HEX(a) FROM t1;
+HEX(a)
+B042
+DROP TABLE t1;
+#
+# End of 10.2 tests
+#
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result
index 816fe654028..f52e08a676f 100644
--- a/mysql-test/r/ctype_utf8.result
+++ b/mysql-test/r/ctype_utf8.result
@@ -10401,3 +10401,30 @@ SET @@SQL_MODE=default;
 #
 # End of 10.1 tests
 #
+#
+# Start of 10.2 tests
+#
+#
+# MDEV-9824 LOAD DATA does not work with multi-byte strings in LINES TERMINATED BY when IGNORE is specified
+#
+CREATE TABLE t1 (c1 VARCHAR(10) CHARACTER SET utf8);
+LOAD DATA INFILE '../../std_data/loaddata/mdev9824.txt' INTO TABLE t1 CHARACTER SET utf8 LINES TERMINATED BY 'Ñ‘Ñ‘';
+Warnings:
+Warning	1638	Non-ASCII separator arguments are not fully supported
+SELECT c1 FROM t1;
+c1
+a
+b
+c
+DELETE FROM t1;
+LOAD DATA INFILE '../../std_data/loaddata/mdev9824.txt' INTO TABLE t1 CHARACTER SET utf8 LINES TERMINATED BY 'Ñ‘Ñ‘' IGNORE 1 LINES;
+Warnings:
+Warning	1638	Non-ASCII separator arguments are not fully supported
+SELECT c1 FROM t1;
+c1
+b
+c
+DROP TABLE t1;
+#
+# End of 10.2 tests
+#
diff --git a/mysql-test/std_data/loaddata/mdev8711.txt b/mysql-test/std_data/loaddata/mdev8711.txt
new file mode 100644
index 00000000000..49296a7134d
--- /dev/null
+++ b/mysql-test/std_data/loaddata/mdev8711.txt
@@ -0,0 +1 @@
+°@a°A@°B@
\ No newline at end of file
diff --git a/mysql-test/std_data/loaddata/mdev9824.txt b/mysql-test/std_data/loaddata/mdev9824.txt
new file mode 100644
index 00000000000..7050e081844
--- /dev/null
+++ b/mysql-test/std_data/loaddata/mdev9824.txt
@@ -0,0 +1 @@
+aёёbёёcёё
\ No newline at end of file
diff --git a/mysql-test/t/ctype_gbk.test b/mysql-test/t/ctype_gbk.test
index 07e73cdf745..ae66dbba4b5 100644
--- a/mysql-test/t/ctype_gbk.test
+++ b/mysql-test/t/ctype_gbk.test
@@ -435,3 +435,22 @@ SELECT HEX(CONVERT(CAST(0xA341 AS CHAR CHARACTER SET gb2312) USING utf8));
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9811 LOAD DATA INFILE does not work well with gbk in some cases
+--echo #
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET gbk);
+LOAD DATA INFILE '../../std_data/loaddata/mdev8711.txt' INTO TABLE t1 CHARACTER SET gbk LINES TERMINATED BY '@';
+SELECT HEX(a) FROM t1;
+DELETE FROM t1;
+LOAD DATA INFILE '../../std_data/loaddata/mdev8711.txt' INTO TABLE t1 CHARACTER SET gbk LINES TERMINATED BY '@' IGNORE 1 LINES;
+SELECT HEX(a) FROM t1;
+DROP TABLE t1;
+
+--echo #
+--echo # End of 10.2 tests
+--echo #
diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test
index 85ffed943cf..f3a9e63b57d 100644
--- a/mysql-test/t/ctype_utf8.test
+++ b/mysql-test/t/ctype_utf8.test
@@ -1950,3 +1950,22 @@ SET @@SQL_MODE=default;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+--echo #
+--echo # Start of 10.2 tests
+--echo #
+
+--echo #
+--echo # MDEV-9824 LOAD DATA does not work with multi-byte strings in LINES TERMINATED BY when IGNORE is specified
+--echo #
+CREATE TABLE t1 (c1 VARCHAR(10) CHARACTER SET utf8);
+LOAD DATA INFILE '../../std_data/loaddata/mdev9824.txt' INTO TABLE t1 CHARACTER SET utf8 LINES TERMINATED BY 'Ñ‘Ñ‘';
+SELECT c1 FROM t1;
+DELETE FROM t1;
+LOAD DATA INFILE '../../std_data/loaddata/mdev9824.txt' INTO TABLE t1 CHARACTER SET utf8 LINES TERMINATED BY 'Ñ‘Ñ‘' IGNORE 1 LINES;
+SELECT c1 FROM t1;
+DROP TABLE t1;
+
+--echo #
+--echo # End of 10.2 tests
+--echo #
diff --git a/mysys/charset.c b/mysys/charset.c
index ad3eb78ae0e..3c134dc388e 100644
--- a/mysys/charset.c
+++ b/mysys/charset.c
@@ -545,6 +545,7 @@ static void init_available_charsets(void)
   {
     if (*cs)
     {
+      DBUG_ASSERT(cs[0]->mbmaxlen <= MY_CS_MBMAXLEN);
       if (cs[0]->ctype)
         if (init_state_maps(*cs))
           *cs= NULL;
diff --git a/sql/sql_load.cc b/sql/sql_load.cc
index d43eb884abd..f1c29203f3e 100644
--- a/sql/sql_load.cc
+++ b/sql/sql_load.cc
@@ -79,6 +79,14 @@ class READ_INFO {
   NET *io_net;
   int level; /* for load xml */
 
+  bool getbyte(char *to)
+  {
+    int chr= GET;
+    if (chr == my_b_EOF)
+      return (eof= true);
+    *to= chr;
+    return false;
+  }
 public:
   bool error,line_cuted,found_null,enclosed;
   uchar	*row_start,			/* Found row starts here */
@@ -1706,33 +1714,76 @@ int READ_INFO::next_line()
     return 0;					// No lines
   for (;;)
   {
-    int chr = GET;
-#ifdef USE_MB
-   if (my_mbcharlen(read_charset, chr) > 1)
-   {
-       for (uint i=1;
-            chr != my_b_EOF && i<my_mbcharlen(read_charset, chr);
-            i++)
-	   chr = GET;
-       if (chr == escape_char)
-	   continue;
-   }
-#endif
-   if (chr == my_b_EOF)
-   {
-      eof=1;
-      return 1;
+    int chlen;
+    char buf[MY_CS_MBMAXLEN];
+
+    if (getbyte(&buf[0]))
+      return 1; // EOF
+
+    if (use_mb(read_charset) &&
+        (chlen= my_charlen(read_charset, buf, buf + 1)) != 1)
+    {
+      uint i;
+      for (i= 1; MY_CS_IS_TOOSMALL(chlen); )
+      {
+        DBUG_ASSERT(i < sizeof(buf));
+        DBUG_ASSERT(chlen != 1);
+        if (getbyte(&buf[i++]))
+          return 1; // EOF
+        chlen= my_charlen(read_charset, buf, buf + i);
+      }
+
+      /*
+        Either a complete multi-byte sequence,
+        or a broken byte sequence was found.
+        Check if the sequence is a prefix of the "LINES TERMINATED BY" string.
+      */
+      if ((uchar) buf[0] == line_term_char && i <= line_term_length &&
+          !memcmp(buf, line_term_ptr, i))
+      {
+        if (line_term_length == i)
+        {
+          /*
+            We found a "LINES TERMINATED BY" string that consists
+            of a single multi-byte character.
+          */
+          return 0;
+        }
+        /*
+          buf[] is a prefix of "LINES TERMINATED BY".
+          Now check the suffix. Length of the suffix of line_term_ptr
+          that still needs to be checked is (line_term_length - i).
+          Note, READ_INFO::terminator() assumes that the leftmost byte of the
+          argument is already scanned from the file and is checked to
+          be a known prefix (e.g. against line_term_char).
+          So we need to pass one extra byte.
+        */
+        if (terminator(line_term_ptr + i - 1, line_term_length - i + 1))
+          return 0;
+      }
+      /*
+        Here we have a good multi-byte sequence or a broken byte sequence,
+        and the sequence is not equal to "LINES TERMINATED BY".
+        No needs to check for escape_char, because:
+        - multi-byte escape characters in "FIELDS ESCAPED BY" are not
+          supported and are rejected at parse time.
+        - broken single-byte sequences are not recognized as escapes,
+          they are considered to be a part of the data and are converted to
+          question marks.
+      */
+      line_cuted= true;
+      continue;
     }
-    if (chr == escape_char)
+    if (buf[0] == escape_char)
     {
-      line_cuted=1;
+      line_cuted= true;
       if (GET == my_b_EOF)
-	return 1;
+        return 1;
       continue;
     }
-    if (chr == line_term_char && terminator(line_term_ptr,line_term_length))
+    if (buf[0] == line_term_char && terminator(line_term_ptr,line_term_length))
       return 0;
-    line_cuted=1;
+    line_cuted= true;
   }
 }