N'xxx' and _utf8'xxx' are not equivalent

Problem: Unescaping of '\' characters didn't work when processing N'xxx'.
Fix: using get_text() instead of get_token() when scanning nationa strings.


mysql-test/r/ctype_utf8.result:
  Adding test case
mysql-test/t/ctype_utf8.test:
  Adding test case
sql/sql_lex.cc:
  Fixing to process national strings using get_tex(),
  i.e. the same way with usual strings, to make
  unescaping work.

1 files changed, 9 insertions, 12 deletions

diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index 7d4dca15608..5a355cedca5 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -557,23 +557,20 @@ int MYSQLlex(void *arg, void *yythd)
 
     case MY_LEX_IDENT_OR_NCHAR:
       if (yyPeek() != '\'')
-      {					// Found x'hex-number'
+      {					
 	state= MY_LEX_IDENT;
 	break;
       }
-      yyGet();				// Skip '
-      while ((c = yyGet()) && (c !='\'')) ;
-      length=(lex->ptr - lex->tok_start);	// Length of hexnum+3
-      if (c != '\'')
+      /* Found N'string' */
+      lex->tok_start++;                 // Skip N
+      yySkip();                         // Skip '
+      if (!(yylval->lex_str.str = get_text(lex)))
       {
-	return(ABORT_SYM);		// Illegal hex constant
+	state= MY_LEX_CHAR;             // Read char by char
+	break;
       }
-      yyGet();				// get_token makes an unget
-      yylval->lex_str=get_token(lex,length);
-      yylval->lex_str.str+=2;		// Skip x'
-      yylval->lex_str.length-=3;	// Don't count x' and last '
-      lex->yytoklen-=3;
-      return (NCHAR_STRING);
+      yylval->lex_str.length= lex->yytoklen;
+      return(NCHAR_STRING);
 
     case MY_LEX_IDENT_OR_HEX:
       if (yyPeek() == '\'')