Bug#25411 (trigger code truncated), PART I

The issue found with bug 25411 is due to the function skip_rear_comments()
which damages the source code while implementing a work around.
The root cause of the problem is in the lexical analyser, which does not
process special comments properly.
For special comments like :
[1] aaa /*!50000 bbb */ ccc
since 5.0 is a version older that the current code, the parser is in lining
the content of the special comment, so that the query to process is
[2] aaa bbb ccc
However, the text of the query captured when processing a stored procedure,
stored function or trigger (or event in 5.1), can be after rebuilding it:
[3] aaa bbb */ ccc
which is wrong.

To fix bug 25411 properly, the lexical analyser needs to return [2] when
in lining special comments.
In order to implement this, some preliminary cleanup is required in the code,
which is implemented by this patch.

Before this change, the structure named LEX (or st_lex) contains attributes
that belong to lexical analysis, as well as attributes that represents the
abstract syntax tree (AST) of a statement.
Creating a new LEX structure for each statements (which makes sense for the
AST part) also re-initialized the lexical analysis phase each time, which
is conceptually wrong.

With this patch, the previous st_lex structure has been split in two:
- st_lex represents the Abstract Syntax Tree for a statement. The name "lex"
has not been changed to avoid a bigger impact in the code base.
- class lex_input_stream represents the internal state of the lexical
  analyser, which by definition should *not* be reinitialized when parsing
  multiple statements from the same input stream.

This change is a pre-requisite for bug 25411, since the implementation of
lex_input_stream will later improve to deal properly with special comments,
and this processing can not be done with the current implementation of
sp_head::reset_lex and sp_head::restore_lex, which interfere with the lexer.

This change set alone does not fix bug 25411.


sql/item_func.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/log_event.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/mysql_priv.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/slave.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp_head.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp_head.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_class.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_class.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_lex.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_lex.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_parse.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_prepare.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_trigger.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_view.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_yacc.yy:
  Refactoring, separate lex_input_stream from st_lex.

1 files changed, 159 insertions, 140 deletions

diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index 3be844b6761..6fe4bfbd4f1 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -32,13 +32,13 @@ sys_var_long_ptr trg_new_row_fake_var(0, 0);
 
 /* Macros to look like lex */
 
-#define yyGet()		*(lex->ptr++)
-#define yyGetLast()	lex->ptr[-1]
-#define yyPeek()	lex->ptr[0]
-#define yyPeek2()	lex->ptr[1]
-#define yyUnget()	lex->ptr--
-#define yySkip()	lex->ptr++
-#define yyLength()	((uint) (lex->ptr - lex->tok_start)-1)
+#define yyGet()		*(lip->ptr++)
+#define yyGetLast()	lip->ptr[-1]
+#define yyPeek()	lip->ptr[0]
+#define yyPeek2()	lip->ptr[1]
+#define yyUnget()	lip->ptr--
+#define yySkip()	lip->ptr++
+#define yyLength()	((uint) (lip->ptr - lip->tok_start)-1)
 
 /* Longest standard keyword name */
 #define TOCK_NAME_LENGTH 24
@@ -108,6 +108,27 @@ st_parsing_options::reset()
   allows_derived= TRUE;
 }
 
+Lex_input_stream::Lex_input_stream(THD *thd,
+                                   const char* buffer,
+                                   unsigned int length)
+: m_thd(thd),
+  yylineno(1),
+  yytoklen(0),
+  yylval(NULL),
+  ptr(buffer),
+  tok_start(NULL),
+  tok_end(NULL),
+  end_of_query(buffer + length),
+  tok_start_prev(NULL),
+  buf(buffer),
+  next_state(MY_LEX_START),
+  found_semicolon(NULL)
+{
+}
+
+Lex_input_stream::~Lex_input_stream()
+{}
+
 
 /*
   This is called before every query that is to be parsed.
@@ -115,14 +136,12 @@ st_parsing_options::reset()
   (We already do too much here)
 */
 
-void lex_start(THD *thd, uchar *buf,uint length)
+void lex_start(THD *thd)
 {
   LEX *lex= thd->lex;
   DBUG_ENTER("lex_start");
 
   lex->thd= lex->unit.thd= thd;
-  lex->buf= lex->ptr= buf;
-  lex->end_of_query= buf+length;
 
   lex->context_stack.empty();
   lex->unit.init_query();
@@ -155,15 +174,13 @@ void lex_start(THD *thd, uchar *buf,uint length)
   lex->stmt_prepare_mode= FALSE;
   lex->derived_tables= 0;
   lex->lock_option= TL_READ;
-  lex->found_semicolon= 0;
   lex->safe_to_cache_query= 1;
   lex->time_zone_tables_used= 0;
   lex->leaf_tables_insert= 0;
   lex->parsing_options.reset();
   lex->empty_field_list_on_rset= 0;
   lex->select_lex.select_number= 1;
-  lex->next_state=MY_LEX_START;
-  lex->yylineno = 1;
+
   lex->in_comment=0;
   lex->length=0;
   lex->select_lex.in_sum_expr=0;
@@ -201,22 +218,22 @@ void lex_end(LEX *lex)
 }
 
 
-static int find_keyword(LEX *lex, uint len, bool function)
+static int find_keyword(Lex_input_stream *lip, uint len, bool function)
 {
-  uchar *tok=lex->tok_start;
+  const char *tok= lip->tok_start;
 
-  SYMBOL *symbol = get_hash_symbol((const char *)tok,len,function);
+  SYMBOL *symbol= get_hash_symbol(tok, len, function);
   if (symbol)
   {
-    lex->yylval->symbol.symbol=symbol;
-    lex->yylval->symbol.str= (char*) tok;
-    lex->yylval->symbol.length=len;
+    lip->yylval->symbol.symbol=symbol;
+    lip->yylval->symbol.str= (char*) tok;
+    lip->yylval->symbol.length=len;
     
     if ((symbol->tok == NOT_SYM) &&
-        (lex->thd->variables.sql_mode & MODE_HIGH_NOT_PRECEDENCE))
+        (lip->m_thd->variables.sql_mode & MODE_HIGH_NOT_PRECEDENCE))
       return NOT2_SYM;
     if ((symbol->tok == OR_OR_SYM) &&
-	!(lex->thd->variables.sql_mode & MODE_PIPES_AS_CONCAT))
+	!(lip->m_thd->variables.sql_mode & MODE_PIPES_AS_CONCAT))
       return OR2_SYM;
     
     return symbol->tok;
@@ -245,12 +262,12 @@ bool is_keyword(const char *name, uint len)
 
 /* make a copy of token before ptr and set yytoklen */
 
-static LEX_STRING get_token(LEX *lex,uint length)
+static LEX_STRING get_token(Lex_input_stream *lip, uint length)
 {
   LEX_STRING tmp;
   yyUnget();			// ptr points now after last token char
-  tmp.length=lex->yytoklen=length;
-  tmp.str=(char*) lex->thd->strmake((char*) lex->tok_start,tmp.length);
+  tmp.length=lip->yytoklen=length;
+  tmp.str= lip->m_thd->strmake(lip->tok_start, tmp.length);
   return tmp;
 }
 
@@ -261,14 +278,15 @@ static LEX_STRING get_token(LEX *lex,uint length)
    future to operate multichar strings (like ucs2)
 */
 
-static LEX_STRING get_quoted_token(LEX *lex,uint length, char quote)
+static LEX_STRING get_quoted_token(Lex_input_stream *lip,
+                                   uint length, char quote)
 {
   LEX_STRING tmp;
   byte *from, *to, *end;
   yyUnget();			// ptr points now after last token char
-  tmp.length=lex->yytoklen=length;
-  tmp.str=(char*) lex->thd->alloc(tmp.length+1);
-  for (from= (byte*) lex->tok_start, to= (byte*) tmp.str, end= to+length ;
+  tmp.length=lip->yytoklen=length;
+  tmp.str=(char*) lip->m_thd->alloc(tmp.length+1);
+  for (from= (byte*) lip->tok_start, to= (byte*) tmp.str, end= to+length ;
        to != end ;
        )
   {
@@ -285,15 +303,14 @@ static LEX_STRING get_quoted_token(LEX *lex,uint length, char quote)
   Fix sometimes to do only one scan of the string
 */
 
-static char *get_text(LEX *lex)
+static char *get_text(Lex_input_stream *lip)
 {
   reg1 uchar c,sep;
   uint found_escape=0;
-  CHARSET_INFO *cs= lex->thd->charset();
+  CHARSET_INFO *cs= lip->m_thd->charset();
 
   sep= yyGetLast();			// String should end with this
-  //lex->tok_start=lex->ptr-1;		// Remember '
-  while (lex->ptr != lex->end_of_query)
+  while (lip->ptr != lip->end_of_query)
   {
     c = yyGet();
 #ifdef USE_MB
@@ -301,18 +318,18 @@ static char *get_text(LEX *lex)
       int l;
       if (use_mb(cs) &&
           (l = my_ismbchar(cs,
-                           (const char *)lex->ptr-1,
-                           (const char *)lex->end_of_query))) {
-	lex->ptr += l-1;
+                           lip->ptr-1,
+                           lip->end_of_query))) {
+	lip->ptr += l-1;
 	continue;
       }
     }
 #endif
     if (c == '\\' &&
-	!(lex->thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES))
+	!(lip->m_thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES))
     {					// Escaped character
       found_escape=1;
-      if (lex->ptr == lex->end_of_query)
+      if (lip->ptr == lip->end_of_query)
 	return 0;
       yySkip();
     }
@@ -327,21 +344,23 @@ static char *get_text(LEX *lex)
 	yyUnget();
 
       /* Found end. Unescape and return string */
-      uchar *str,*end,*start;
+      const char *str;
+      const char *end;
+      char *start;
 
-      str=lex->tok_start+1;
-      end=lex->ptr-1;
-      if (!(start=(uchar*) lex->thd->alloc((uint) (end-str)+1)))
+      str=lip->tok_start+1;
+      end=lip->ptr-1;
+      if (!(start=(char*) lip->m_thd->alloc((uint) (end-str)+1)))
 	return (char*) "";		// Sql_alloc has set error flag
       if (!found_escape)
       {
-	lex->yytoklen=(uint) (end-str);
-	memcpy(start,str,lex->yytoklen);
-	start[lex->yytoklen]=0;
+	lip->yytoklen=(uint) (end-str);
+	memcpy(start,str,lip->yytoklen);
+	start[lip->yytoklen]=0;
       }
       else
       {
-	uchar *to;
+	char *to;
 
 	for (to=start ; str != end ; str++)
 	{
@@ -356,7 +375,7 @@ static char *get_text(LEX *lex)
 	      continue;
 	  }
 #endif
-	  if (!(lex->thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES) &&
+	  if (!(lip->m_thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES) &&
               *str == '\\' && str+1 != end)
 	  {
 	    switch(*++str) {
@@ -393,7 +412,7 @@ static char *get_text(LEX *lex)
 	    *to++ = *str;
 	}
 	*to=0;
-	lex->yytoklen=(uint) (to-start);
+	lip->yytoklen=(uint) (to-start);
       }
       return (char*) start;
     }
@@ -506,20 +525,21 @@ int MYSQLlex(void *arg, void *yythd)
   int	tokval, result_state;
   uint length;
   enum my_lex_states state;
-  LEX	*lex= ((THD *)yythd)->lex;
+  THD *thd= (THD *)yythd;
+  Lex_input_stream *lip= thd->m_lip;
+  LEX *lex= thd->lex;
   YYSTYPE *yylval=(YYSTYPE*) arg;
-  CHARSET_INFO *cs= ((THD *) yythd)->charset();
+  CHARSET_INFO *cs= thd->charset();
   uchar *state_map= cs->state_map;
   uchar *ident_map= cs->ident_map;
 
-  lex->yylval=yylval;			// The global state
+  lip->yylval=yylval;			// The global state
 
-  lex->tok_end_prev= lex->tok_end;
-  lex->tok_start_prev= lex->tok_start;
+  lip->tok_start_prev= lip->tok_start;
 
-  lex->tok_start=lex->tok_end=lex->ptr;
-  state=lex->next_state;
-  lex->next_state=MY_LEX_OPERATOR_OR_IDENT;
+  lip->tok_start=lip->tok_end=lip->ptr;
+  state=lip->next_state;
+  lip->next_state=MY_LEX_OPERATOR_OR_IDENT;
   LINT_INIT(c);
   for (;;)
   {
@@ -530,9 +550,9 @@ int MYSQLlex(void *arg, void *yythd)
       for (c=yyGet() ; (state_map[c] == MY_LEX_SKIP) ; c= yyGet())
       {
 	if (c == '\n')
-	  lex->yylineno++;
+	  lip->yylineno++;
       }
-      lex->tok_start=lex->ptr-1;	// Start of real token
+      lip->tok_start=lip->ptr-1;	// Start of real token
       state= (enum my_lex_states) state_map[c];
       break;
     case MY_LEX_ESCAPE:
@@ -551,13 +571,13 @@ int MYSQLlex(void *arg, void *yythd)
         state=MY_LEX_COMMENT;
         break;
       }
-      yylval->lex_str.str=(char*) (lex->ptr=lex->tok_start);// Set to first chr
+      yylval->lex_str.str=(char*) (lip->ptr=lip->tok_start);// Set to first chr
       yylval->lex_str.length=1;
       c=yyGet();
       if (c != ')')
-	lex->next_state= MY_LEX_START;	// Allow signed numbers
+	lip->next_state= MY_LEX_START;	// Allow signed numbers
       if (c == ',')
-	lex->tok_start=lex->ptr;	// Let tok_start point at next item
+	lip->tok_start=lip->ptr;	// Let tok_start point at next item
       /*
         Check for a placeholder: it should not precede a possible identifier
         because of binlogging: when a placeholder is replaced with
@@ -575,14 +595,14 @@ int MYSQLlex(void *arg, void *yythd)
 	break;
       }
       /* Found N'string' */
-      lex->tok_start++;                 // Skip N
+      lip->tok_start++;                 // Skip N
       yySkip();                         // Skip '
-      if (!(yylval->lex_str.str = get_text(lex)))
+      if (!(yylval->lex_str.str = get_text(lip)))
       {
 	state= MY_LEX_CHAR;             // Read char by char
 	break;
       }
-      yylval->lex_str.length= lex->yytoklen;
+      yylval->lex_str.length= lip->yytoklen;
       return(NCHAR_STRING);
 
     case MY_LEX_IDENT_OR_HEX:
@@ -598,7 +618,7 @@ int MYSQLlex(void *arg, void *yythd)
         break;
       }
     case MY_LEX_IDENT:
-      uchar *start;
+      const char *start;
 #if defined(USE_MB) && defined(USE_MB_IDENT)
       if (use_mb(cs))
       {
@@ -606,13 +626,13 @@ int MYSQLlex(void *arg, void *yythd)
         if (my_mbcharlen(cs, yyGetLast()) > 1)
         {
           int l = my_ismbchar(cs,
-                              (const char *)lex->ptr-1,
-                              (const char *)lex->end_of_query);
+                              lip->ptr-1,
+                              lip->end_of_query);
           if (l == 0) {
             state = MY_LEX_CHAR;
             continue;
           }
-          lex->ptr += l - 1;
+          lip->ptr += l - 1;
         }
         while (ident_map[c=yyGet()])
         {
@@ -620,10 +640,10 @@ int MYSQLlex(void *arg, void *yythd)
           {
             int l;
             if ((l = my_ismbchar(cs,
-                              (const char *)lex->ptr-1,
-                              (const char *)lex->end_of_query)) == 0)
+                                 lip->ptr-1,
+                                 lip->end_of_query)) == 0)
               break;
-            lex->ptr += l-1;
+            lip->ptr += l-1;
           }
         }
       }
@@ -634,8 +654,8 @@ int MYSQLlex(void *arg, void *yythd)
         /* If there were non-ASCII characters, mark that we must convert */
         result_state= result_state & 0x80 ? IDENT_QUOTED : IDENT;
       }
-      length= (uint) (lex->ptr - lex->tok_start)-1;
-      start= lex->ptr;
+      length= (uint) (lip->ptr - lip->tok_start)-1;
+      start= lip->ptr;
       if (lex->ignore_space)
       {
         /*
@@ -644,19 +664,19 @@ int MYSQLlex(void *arg, void *yythd)
         */
         for (; state_map[c] == MY_LEX_SKIP ; c= yyGet());
       }
-      if (start == lex->ptr && c == '.' && ident_map[yyPeek()])
-	lex->next_state=MY_LEX_IDENT_SEP;
+      if (start == lip->ptr && c == '.' && ident_map[yyPeek()])
+	lip->next_state=MY_LEX_IDENT_SEP;
       else
       {					// '(' must follow directly if function
 	yyUnget();
-	if ((tokval = find_keyword(lex,length,c == '(')))
+	if ((tokval = find_keyword(lip, length, c == '(')))
 	{
-	  lex->next_state= MY_LEX_START;	// Allow signed numbers
+	  lip->next_state= MY_LEX_START;	// Allow signed numbers
 	  return(tokval);		// Was keyword
 	}
 	yySkip();			// next state does a unget
       }
-      yylval->lex_str=get_token(lex,length);
+      yylval->lex_str=get_token(lip, length);
 
       /* 
          Note: "SELECT _bla AS 'alias'"
@@ -673,12 +693,12 @@ int MYSQLlex(void *arg, void *yythd)
       return(result_state);			// IDENT or IDENT_QUOTED
 
     case MY_LEX_IDENT_SEP:		// Found ident and now '.'
-      yylval->lex_str.str=(char*) lex->ptr;
+      yylval->lex_str.str=(char*) lip->ptr;
       yylval->lex_str.length=1;
       c=yyGet();			// should be '.'
-      lex->next_state= MY_LEX_IDENT_START;// Next is an ident (not a keyword)
+      lip->next_state= MY_LEX_IDENT_START;// Next is an ident (not a keyword)
       if (!ident_map[yyPeek()])		// Probably ` or "
-	lex->next_state= MY_LEX_START;
+	lip->next_state= MY_LEX_START;
       return((int) c);
 
     case MY_LEX_NUMBER_IDENT:		// number or ident which num-start
@@ -698,36 +718,36 @@ int MYSQLlex(void *arg, void *yythd)
 	  {
 	    yySkip();
 	    while (my_isdigit(cs,yyGet())) ;
-	    yylval->lex_str=get_token(lex,yyLength());
+	    yylval->lex_str=get_token(lip, yyLength());
 	    return(FLOAT_NUM);
 	  }
 	}
 	yyUnget(); /* purecov: inspected */
       }
-      else if (c == 'x' && (lex->ptr - lex->tok_start) == 2 &&
-	  lex->tok_start[0] == '0' )
+      else if (c == 'x' && (lip->ptr - lip->tok_start) == 2 &&
+	  lip->tok_start[0] == '0' )
       {						// Varbinary
 	while (my_isxdigit(cs,(c = yyGet()))) ;
-	if ((lex->ptr - lex->tok_start) >= 4 && !ident_map[c])
+	if ((lip->ptr - lip->tok_start) >= 4 && !ident_map[c])
 	{
-	  yylval->lex_str=get_token(lex,yyLength());
+	  yylval->lex_str=get_token(lip, yyLength());
 	  yylval->lex_str.str+=2;		// Skip 0x
 	  yylval->lex_str.length-=2;
-	  lex->yytoklen-=2;
+	  lip->yytoklen-=2;
 	  return (HEX_NUM);
 	}
 	yyUnget();
       }
-      else if (c == 'b' && (lex->ptr - lex->tok_start) == 2 &&
-               lex->tok_start[0] == '0' )
+      else if (c == 'b' && (lip->ptr - lip->tok_start) == 2 &&
+               lip->tok_start[0] == '0' )
       {						// b'bin-number'
 	while (my_isxdigit(cs,(c = yyGet()))) ;
-	if ((lex->ptr - lex->tok_start) >= 4 && !ident_map[c])
+	if ((lip->ptr - lip->tok_start) >= 4 && !ident_map[c])
 	{
-	  yylval->lex_str= get_token(lex, yyLength());
+	  yylval->lex_str= get_token(lip, yyLength());
 	  yylval->lex_str.str+= 2;		// Skip 0x
 	  yylval->lex_str.length-= 2;
-	  lex->yytoklen-= 2;
+	  lip->yytoklen-= 2;
 	  return (BIN_NUM);
 	}
 	yyUnget();
@@ -745,10 +765,10 @@ int MYSQLlex(void *arg, void *yythd)
           {
             int l;
             if ((l = my_ismbchar(cs,
-                                 (const char *)lex->ptr-1,
-                                 (const char *)lex->end_of_query)) == 0)
+                                 lip->ptr-1,
+                                 lip->end_of_query)) == 0)
               break;
-            lex->ptr += l-1;
+            lip->ptr += l-1;
           }
         }
       }
@@ -760,16 +780,16 @@ int MYSQLlex(void *arg, void *yythd)
         result_state= result_state & 0x80 ? IDENT_QUOTED : IDENT;
       }
       if (c == '.' && ident_map[yyPeek()])
-	lex->next_state=MY_LEX_IDENT_SEP;// Next is '.'
+	lip->next_state=MY_LEX_IDENT_SEP;// Next is '.'
 
-      yylval->lex_str= get_token(lex,yyLength());
+      yylval->lex_str= get_token(lip, yyLength());
       return(result_state);
 
     case MY_LEX_USER_VARIABLE_DELIMITER:	// Found quote char
     {
       uint double_quotes= 0;
       char quote_char= c;                       // Used char
-      lex->tok_start=lex->ptr;			// Skip first `
+      lip->tok_start=lip->ptr;			// Skip first `
       while ((c=yyGet()))
       {
 	int var_length;
@@ -789,23 +809,23 @@ int MYSQLlex(void *arg, void *yythd)
 #ifdef USE_MB
 	else if (var_length < 1)
 	  break;				// Error
-	lex->ptr+= var_length-1;
+	lip->ptr+= var_length-1;
 #endif
       }
       if (double_quotes)
-	yylval->lex_str=get_quoted_token(lex,yyLength() - double_quotes,
+	yylval->lex_str=get_quoted_token(lip, yyLength() - double_quotes,
 					 quote_char);
       else
-	yylval->lex_str=get_token(lex,yyLength());
+	yylval->lex_str=get_token(lip, yyLength());
       if (c == quote_char)
 	yySkip();			// Skip end `
-      lex->next_state= MY_LEX_START;
+      lip->next_state= MY_LEX_START;
       return(IDENT_QUOTED);
     }
     case MY_LEX_INT_OR_REAL:		// Compleat int or incompleat real
       if (c != '.')
       {					// Found complete integer number.
-	yylval->lex_str=get_token(lex,yyLength());
+	yylval->lex_str=get_token(lip, yyLength());
 	return int_token(yylval->lex_str.str,yylval->lex_str.length);
       }
       // fall through
@@ -823,47 +843,47 @@ int MYSQLlex(void *arg, void *yythd)
 	  break;
 	}
 	while (my_isdigit(cs,yyGet())) ;
-	yylval->lex_str=get_token(lex,yyLength());
+	yylval->lex_str=get_token(lip, yyLength());
 	return(FLOAT_NUM);
       }
-      yylval->lex_str=get_token(lex,yyLength());
+      yylval->lex_str=get_token(lip, yyLength());
       return(DECIMAL_NUM);
 
     case MY_LEX_HEX_NUMBER:		// Found x'hexstring'
       yyGet();				// Skip '
       while (my_isxdigit(cs,(c = yyGet()))) ;
-      length=(lex->ptr - lex->tok_start);	// Length of hexnum+3
+      length=(lip->ptr - lip->tok_start);	// Length of hexnum+3
       if (!(length & 1) || c != '\'')
       {
 	return(ABORT_SYM);		// Illegal hex constant
       }
       yyGet();				// get_token makes an unget
-      yylval->lex_str=get_token(lex,length);
+      yylval->lex_str=get_token(lip, length);
       yylval->lex_str.str+=2;		// Skip x'
       yylval->lex_str.length-=3;	// Don't count x' and last '
-      lex->yytoklen-=3;
+      lip->yytoklen-=3;
       return (HEX_NUM);
 
     case MY_LEX_BIN_NUMBER:           // Found b'bin-string'
       yyGet();                                // Skip '
       while ((c= yyGet()) == '0' || c == '1');
-      length= (lex->ptr - lex->tok_start);    // Length of bin-num + 3
+      length= (lip->ptr - lip->tok_start);    // Length of bin-num + 3
       if (c != '\'')
       return(ABORT_SYM);              // Illegal hex constant
       yyGet();                        // get_token makes an unget
-      yylval->lex_str= get_token(lex, length);
+      yylval->lex_str= get_token(lip, length);
       yylval->lex_str.str+= 2;        // Skip b'
       yylval->lex_str.length-= 3;     // Don't count b' and last '
-      lex->yytoklen-= 3;
+      lip->yytoklen-= 3;
       return (BIN_NUM); 
 
     case MY_LEX_CMP_OP:			// Incomplete comparison operator
       if (state_map[yyPeek()] == MY_LEX_CMP_OP ||
 	  state_map[yyPeek()] == MY_LEX_LONG_CMP_OP)
 	yySkip();
-      if ((tokval = find_keyword(lex,(uint) (lex->ptr - lex->tok_start),0)))
+      if ((tokval = find_keyword(lip,(uint) (lip->ptr - lip->tok_start),0)))
       {
-	lex->next_state= MY_LEX_START;	// Allow signed numbers
+	lip->next_state= MY_LEX_START;	// Allow signed numbers
 	return(tokval);
       }
       state = MY_LEX_CHAR;		// Something fishy found
@@ -877,9 +897,9 @@ int MYSQLlex(void *arg, void *yythd)
 	if (state_map[yyPeek()] == MY_LEX_CMP_OP)
 	  yySkip();
       }
-      if ((tokval = find_keyword(lex,(uint) (lex->ptr - lex->tok_start),0)))
+      if ((tokval = find_keyword(lip,(uint) (lip->ptr - lip->tok_start),0)))
       {
-	lex->next_state= MY_LEX_START;	// Found long op
+	lip->next_state= MY_LEX_START;	// Found long op
 	return(tokval);
       }
       state = MY_LEX_CHAR;		// Something fishy found
@@ -892,24 +912,24 @@ int MYSQLlex(void *arg, void *yythd)
 	break;
       }
       yySkip();
-      tokval = find_keyword(lex,2,0);	// Is a bool operator
-      lex->next_state= MY_LEX_START;	// Allow signed numbers
+      tokval = find_keyword(lip,2,0);	// Is a bool operator
+      lip->next_state= MY_LEX_START;	// Allow signed numbers
       return(tokval);
 
     case MY_LEX_STRING_OR_DELIMITER:
-      if (((THD *) yythd)->variables.sql_mode & MODE_ANSI_QUOTES)
+      if (thd->variables.sql_mode & MODE_ANSI_QUOTES)
       {
 	state= MY_LEX_USER_VARIABLE_DELIMITER;
 	break;
       }
       /* " used for strings */
     case MY_LEX_STRING:			// Incomplete text string
-      if (!(yylval->lex_str.str = get_text(lex)))
+      if (!(yylval->lex_str.str = get_text(lip)))
       {
 	state= MY_LEX_CHAR;		// Read char by char
 	break;
       }
-      yylval->lex_str.length=lex->yytoklen;
+      yylval->lex_str.length=lip->yytoklen;
       return(TEXT_STRING);
 
     case MY_LEX_COMMENT:			//  Comment
@@ -933,7 +953,7 @@ int MYSQLlex(void *arg, void *yythd)
 	state=MY_LEX_START;
 	if (my_isdigit(cs,yyPeek()))
 	{				// Version number
-	  version=strtol((char*) lex->ptr,(char**) &lex->ptr,10);
+	  version=strtol((char*) lip->ptr,(char**) &lip->ptr,10);
 	}
 	if (version <= MYSQL_VERSION_ID)
 	{
@@ -941,13 +961,13 @@ int MYSQLlex(void *arg, void *yythd)
 	  break;
 	}
       }
-      while (lex->ptr != lex->end_of_query &&
+      while (lip->ptr != lip->end_of_query &&
 	     ((c=yyGet()) != '*' || yyPeek() != '/'))
       {
 	if (c == '\n')
-	  lex->yylineno++;
+	  lip->yylineno++;
       }
-      if (lex->ptr != lex->end_of_query)
+      if (lip->ptr != lip->end_of_query)
 	yySkip();			// remove last '/'
       state = MY_LEX_START;		// Try again
       break;
@@ -972,14 +992,13 @@ int MYSQLlex(void *arg, void *yythd)
     case MY_LEX_SEMICOLON:			// optional line terminator
       if (yyPeek())
       {
-        THD* thd= (THD*)yythd;
         if ((thd->client_capabilities & CLIENT_MULTI_STATEMENTS) && 
             !lex->stmt_prepare_mode)
         {
 	  lex->safe_to_cache_query= 0;
-          lex->found_semicolon=(char*) lex->ptr;
+          lip->found_semicolon= lip->ptr;
           thd->server_status|= SERVER_MORE_RESULTS_EXISTS;
-          lex->next_state=     MY_LEX_END;
+          lip->next_state=     MY_LEX_END;
           return (END_OF_INPUT);
         }
         state= MY_LEX_CHAR;		// Return ';'
@@ -987,15 +1006,15 @@ int MYSQLlex(void *arg, void *yythd)
       }
       /* fall true */
     case MY_LEX_EOL:
-      if (lex->ptr >= lex->end_of_query)
+      if (lip->ptr >= lip->end_of_query)
       {
-	lex->next_state=MY_LEX_END;	// Mark for next loop
+	lip->next_state=MY_LEX_END;	// Mark for next loop
 	return(END_OF_INPUT);
       }
       state=MY_LEX_CHAR;
       break;
     case MY_LEX_END:
-      lex->next_state=MY_LEX_END;
+      lip->next_state=MY_LEX_END;
       return(0);			// We found end of input last time
       
       /* Actually real shouldn't start with . but allow them anyhow */
@@ -1015,26 +1034,26 @@ int MYSQLlex(void *arg, void *yythd)
       case MY_LEX_STRING_OR_DELIMITER:
 	break;
       case MY_LEX_USER_END:
-	lex->next_state=MY_LEX_SYSTEM_VAR;
+	lip->next_state=MY_LEX_SYSTEM_VAR;
 	break;
       default:
-	lex->next_state=MY_LEX_HOSTNAME;
+	lip->next_state=MY_LEX_HOSTNAME;
 	break;
       }
-      yylval->lex_str.str=(char*) lex->ptr;
+      yylval->lex_str.str=(char*) lip->ptr;
       yylval->lex_str.length=1;
       return((int) '@');
     case MY_LEX_HOSTNAME:		// end '@' of user@hostname
       for (c=yyGet() ; 
 	   my_isalnum(cs,c) || c == '.' || c == '_' ||  c == '$';
 	   c= yyGet()) ;
-      yylval->lex_str=get_token(lex,yyLength());
+      yylval->lex_str=get_token(lip, yyLength());
       return(LEX_HOSTNAME);
     case MY_LEX_SYSTEM_VAR:
-      yylval->lex_str.str=(char*) lex->ptr;
+      yylval->lex_str.str=(char*) lip->ptr;
       yylval->lex_str.length=1;
       yySkip();					// Skip '@'
-      lex->next_state= (state_map[yyPeek()] ==
+      lip->next_state= (state_map[yyPeek()] ==
 			MY_LEX_USER_VARIABLE_DELIMITER ?
 			MY_LEX_OPERATOR_OR_IDENT :
 			MY_LEX_IDENT_OR_KEYWORD);
@@ -1051,16 +1070,16 @@ int MYSQLlex(void *arg, void *yythd)
       result_state= result_state & 0x80 ? IDENT_QUOTED : IDENT;
       
       if (c == '.')
-	lex->next_state=MY_LEX_IDENT_SEP;
-      length= (uint) (lex->ptr - lex->tok_start)-1;
+	lip->next_state=MY_LEX_IDENT_SEP;
+      length= (uint) (lip->ptr - lip->tok_start)-1;
       if (length == 0) 
         return(ABORT_SYM);              // Names must be nonempty.
-      if ((tokval= find_keyword(lex,length,0)))
+      if ((tokval= find_keyword(lip, length,0)))
       {
 	yyUnget();				// Put back 'c'
 	return(tokval);				// Was keyword
       }
-      yylval->lex_str=get_token(lex,length);
+      yylval->lex_str=get_token(lip, length);
       return(result_state);
     }
   }
@@ -1093,7 +1112,7 @@ Alter_info::Alter_info(const Alter_info &rhs, MEM_ROOT *mem_root)
     Pointer to the last non-comment symbol of the statement.
 */
 
-uchar *skip_rear_comments(uchar *begin, uchar *end)
+char *skip_rear_comments(char *begin, char *end)
 {
   while (begin < end && (end[-1] <= ' ' || end[-1] == '*' ||
                          end[-1] == '/' || end[-1] == ';'))