WL#2575 - Fulltext: Parser plugin for FTS

Manual merge.

10 files changed, 184 insertions, 60 deletions

diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c
index 3a149b68e93..fbd8e03f75c 100644
--- a/storage/myisam/ft_boolean_search.c
+++ b/storage/myisam/ft_boolean_search.c
@@ -284,28 +284,29 @@ static int ftb_parse_query_internal(void *param, byte *query, uint len)
 static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
                              struct st_mysql_ftparser *parser)
 {
-  MYSQL_FTPARSER_PARAM param;
+  MYSQL_FTPARSER_PARAM *param;
   MY_FTB_PARAM ftb_param;
   DBUG_ENTER("_ftb_parse_query");
   DBUG_ASSERT(parser);
 
   if (ftb->state != UNINITIALIZED)
-    return;
+    DBUG_VOID_RETURN;
 
   ftb_param.ftb= ftb;
   ftb_param.depth= 0;
   ftb_param.ftbe= ftb->root;
   ftb_param.up_quot= 0;
 
-  param.mysql_parse= ftb_parse_query_internal;
-  param.mysql_add_word= ftb_query_add_word;
-  param.ftparser_state= 0;
-  param.mysql_ftparam= (void *)&ftb_param;
-  param.cs= ftb->charset;
-  param.doc= query;
-  param.length= len;
-  param.mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
-  parser->parse(&param);
+  if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+    DBUG_VOID_RETURN;
+  param->mysql_parse= ftb_parse_query_internal;
+  param->mysql_add_word= ftb_query_add_word;
+  param->mysql_ftparam= (void *)&ftb_param;
+  param->cs= ftb->charset;
+  param->doc= query;
+  param->length= len;
+  param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
+  parser->parse(param);
   DBUG_VOID_RETURN;
 }
  
@@ -629,30 +630,30 @@ static int ftb_check_phrase_internal(void *param, byte *document, uint len)
     1 is returned if phrase found, 0 else.
 */
 
-static int _ftb_check_phrase(const byte *document, uint len,
-                FTB_EXPR *ftbe, CHARSET_INFO *cs,
-                struct st_mysql_ftparser *parser)
+static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
+                FTB_EXPR *ftbe, struct st_mysql_ftparser *parser)
 {
   MY_FTB_PHRASE_PARAM ftb_param;
-  MYSQL_FTPARSER_PARAM param;
+  MYSQL_FTPARSER_PARAM *param;
   DBUG_ENTER("_ftb_check_phrase");
   DBUG_ASSERT(parser);
+  if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+    DBUG_RETURN(0);
   ftb_param.phrase= ftbe->phrase;
   ftb_param.document= ftbe->document;
-  ftb_param.cs= cs;
+  ftb_param.cs= ftb->charset;
   ftb_param.phrase_length= list_length(ftbe->phrase);
   ftb_param.document_length= 1;
   ftb_param.match= 0;
 
-  param.mysql_parse= ftb_check_phrase_internal;
-  param.mysql_add_word= ftb_phrase_add_word;
-  param.ftparser_state= 0;
-  param.mysql_ftparam= (void *)&ftb_param;
-  param.cs= cs;
-  param.doc= (byte *)document;
-  param.length= len;
-  param.mode= MYSQL_FTPARSER_WITH_STOPWORDS;
-  parser->parse(&param);
+  param->mysql_parse= ftb_check_phrase_internal;
+  param->mysql_add_word= ftb_phrase_add_word;
+  param->mysql_ftparam= (void *)&ftb_param;
+  param->cs= ftb->charset;
+  param->doc= (byte *)document;
+  param->length= len;
+  param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;
+  parser->parse(param);
   DBUG_RETURN(ftb_param.match ? 1 : 0);
 }
 
@@ -696,8 +697,8 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
           {
             if (!ftsi.pos)
               continue;
-            not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.len,
-                                      ftbe, ftb->charset, parser);
+            not_found = ! _ftb_check_phrase(ftb, ftsi.pos, ftsi.len,
+                                            ftbe, parser);
           }
           if (not_found) break;
         } /* ftbe->quot */
@@ -861,7 +862,7 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
   FT_SEG_ITERATOR ftsi, ftsi2;
   my_off_t  docid=ftb->info->lastpos;
   MY_FTB_FIND_PARAM ftb_param;
-  MYSQL_FTPARSER_PARAM param;
+  MYSQL_FTPARSER_PARAM *param;
   struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
                                     &ft_default_parser :
                                     ftb->info->s->keyinfo[ftb->keynr].parser;
@@ -870,6 +871,8 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
     return -2.0;
   if (!ftb->queue.elements)
     return 0;
+  if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+    return 0;
 
   if (ftb->state != INDEX_SEARCH && docid <= ftb->lastpos)
   {
@@ -894,20 +897,20 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
 
   ftb_param.ftb= ftb;
   ftb_param.ftsi= &ftsi2;
-  param.mysql_parse= ftb_find_relevance_parse;
-  param.mysql_add_word= ftb_find_relevance_add_word;
-  param.ftparser_state= 0;
-  param.mysql_ftparam= (void *)&ftb_param;
-  param.cs= ftb->charset;
-  param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
   while (_mi_ft_segiterator(&ftsi))
   {
     if (!ftsi.pos)
       continue;
-
-    param.doc= (byte *)ftsi.pos;
-    param.length= ftsi.len;
-    parser->parse(&param);
+    /* Since subsequent call to _ftb_check_phrase overwrites param elements,
+       it must be reinitialized at each iteration _inside_ the loop. */
+    param->mysql_parse= ftb_find_relevance_parse;
+    param->mysql_add_word= ftb_find_relevance_add_word;
+    param->mysql_ftparam= (void *)&ftb_param;
+    param->cs= ftb->charset;
+    param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
+    param->doc= (byte *)ftsi.pos;
+    param->length= ftsi.len;
+    parser->parse(param);
   }
   ftbe=ftb->root;
   if (ftbe->docid[1]==docid && ftbe->cur_weight>0 &&
diff --git a/storage/myisam/ft_nlq_search.c b/storage/myisam/ft_nlq_search.c
index 82857100d23..b4468d8bd95 100644
--- a/storage/myisam/ft_nlq_search.c
+++ b/storage/myisam/ft_nlq_search.c
@@ -210,6 +210,8 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
   FT_DOC     *dptr;
   FT_INFO    *dlist=NULL;
   my_off_t    saved_lastpos=info->lastpos;
+  struct st_mysql_ftparser *parser;
+  MYSQL_FTPARSER_PARAM *ftparser_param;
   DBUG_ENTER("ft_init_nlq_search");
 
 /* black magic ON */
@@ -223,6 +225,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
   aio.keynr=keynr;
   aio.charset=info->s->keyinfo[keynr].seg->charset;
   aio.keybuff=info->lastkey+info->s->base.max_key_length;
+  parser= info->s->keyinfo[keynr].parser;
+  if (! (ftparser_param= ftparser_call_initializer(info, keynr)))
+    goto err;
 
   bzero(&wtree,sizeof(wtree));
 
@@ -230,7 +235,7 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
             NULL, NULL);
 
   ft_parse_init(&wtree, aio.charset);
-  if (ft_parse(&wtree, query, query_len, 0, info->s->keyinfo[keynr].parser))
+  if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))
     goto err;
 
   if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
@@ -250,7 +255,7 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
       if (!(*info->read_record)(info,docid,record))
       {
         info->update|= HA_STATE_AKTIV;
-        _mi_ft_parse(&wtree, info, keynr, record,1);
+        _mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);
       }
     }
     delete_queue(&best);
diff --git a/storage/myisam/ft_parser.c b/storage/myisam/ft_parser.c
index fb2ace0fa07..f333a661ea9 100644
--- a/storage/myisam/ft_parser.c
+++ b/storage/myisam/ft_parser.c
@@ -284,22 +284,110 @@ static int ft_parse_internal(void *param, byte *doc, uint doc_len)
 
 
 int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
-                    struct st_mysql_ftparser *parser)
+                    struct st_mysql_ftparser *parser,
+                    MYSQL_FTPARSER_PARAM *param)
 {
-  MYSQL_FTPARSER_PARAM param;
   MY_FT_PARSER_PARAM my_param;
   DBUG_ENTER("ft_parse");
   DBUG_ASSERT(parser);
   my_param.wtree= wtree;
   my_param.with_alloc= with_alloc;
 
-  param.mysql_parse= ft_parse_internal;
-  param.mysql_add_word= ft_add_word;
-  param.ftparser_state= 0;
-  param.mysql_ftparam= &my_param;
-  param.cs= wtree->custom_arg;
-  param.doc= doc;
-  param.length= doclen;
-  param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
-  DBUG_RETURN(parser->parse(&param));
+  param->mysql_parse= ft_parse_internal;
+  param->mysql_add_word= ft_add_word;
+  param->mysql_ftparam= &my_param;
+  param->cs= wtree->custom_arg;
+  param->doc= doc;
+  param->length= doclen;
+  param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
+  DBUG_RETURN(parser->parse(param));  
+}
+
+
+MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, uint keynr)
+{
+  uint32 ftparser_nr;
+  struct st_mysql_ftparser *parser;
+  if (! info->ftparser_param)
+  {
+    /* info->ftparser_param can not be zero after the initialization,
+       because it always includes built-in fulltext parser. And built-in
+       parser can be called even if the table has no fulltext indexes and
+       no varchar/text fields. */
+    if (! info->s->ftparsers)
+    {
+      /* It's ok that modification to shared structure is done w/o mutex
+         locks, because all threads would set the same variables to the
+         same values. */
+      uint i, j, keys= info->s->state.header.keys, ftparsers= 1;
+      for (i= 0; i < keys; i++)
+      {
+        MI_KEYDEF *keyinfo= &info->s->keyinfo[i];
+        if (keyinfo->flag & HA_FULLTEXT)
+        {
+          for (j= 0;; j++)
+          {
+            if (j == i)
+            {
+              keyinfo->ftparser_nr= ftparsers++;
+              break;
+            }
+            if (info->s->keyinfo[j].flag & HA_FULLTEXT &&
+                keyinfo->parser == info->s->keyinfo[j].parser)
+            {
+              keyinfo->ftparser_nr= info->s->keyinfo[j].ftparser_nr;
+              break;
+            }
+          }
+        }
+      }
+      info->s->ftparsers= ftparsers;
+    }
+    info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
+      my_malloc(sizeof(MYSQL_FTPARSER_PARAM) *
+                info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));
+    if (! info->ftparser_param)
+      return 0;
+  }
+  if (keynr == NO_SUCH_KEY)
+  {
+    ftparser_nr= 0;
+    parser= &ft_default_parser;
+  }
+  else
+  {
+    ftparser_nr= info->s->keyinfo[keynr].ftparser_nr;
+    parser= info->s->keyinfo[keynr].parser;
+  }
+  if (! info->ftparser_param[ftparser_nr].mysql_add_word)
+  {
+    /* Note, that mysql_add_word is used here as a flag:
+       mysql_add_word == 0 - parser is not initialized
+       mysql_add_word != 0 - parser is initialized, or no
+                             initialization needed. */
+    info->ftparser_param[ftparser_nr].mysql_add_word= (void *)1;
+    if (parser->init && parser->init(&info->ftparser_param[ftparser_nr]))
+      return 0;
+  }
+  return &info->ftparser_param[ftparser_nr];
+}
+
+
+void ftparser_call_deinitializer(MI_INFO *info)
+{
+  uint i, keys= info->s->state.header.keys;
+  if (! info->ftparser_param)
+    return;
+  for (i= 0; i < keys; i++)
+  {
+    MI_KEYDEF *keyinfo= &info->s->keyinfo[i];
+    MYSQL_FTPARSER_PARAM *ftparser_param=
+      &info->ftparser_param[keyinfo->ftparser_nr];
+    if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word)
+    {
+      if (keyinfo->parser->deinit)
+        keyinfo->parser->deinit(ftparser_param);
+      ftparser_param->mysql_add_word= 0;
+    }
+  }
 }
diff --git a/storage/myisam/ft_update.c b/storage/myisam/ft_update.c
index 623419bd701..a280a98cf90 100644
--- a/storage/myisam/ft_update.c
+++ b/storage/myisam/ft_update.c
@@ -96,7 +96,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
 /* parses a document i.e. calls ft_parse for every keyseg */
 
 uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
-                  const byte *record, my_bool with_alloc)
+                  const byte *record, my_bool with_alloc,
+                  MYSQL_FTPARSER_PARAM *param)
 {
   FT_SEG_ITERATOR ftsi;
   struct st_mysql_ftparser *parser;
@@ -109,7 +110,8 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
   while (_mi_ft_segiterator(&ftsi))
   {
     if (ftsi.pos)
-      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser))
+      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,
+                   param))
         DBUG_RETURN(1);
   }
   DBUG_RETURN(0);
@@ -118,10 +120,12 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
 FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
 {
   TREE ptree;
+  MYSQL_FTPARSER_PARAM *param;
   DBUG_ENTER("_mi_ft_parserecord");
-
+  if (! (param= ftparser_call_initializer(info, keynr)))
+    DBUG_RETURN(NULL);
   bzero((char*) &ptree, sizeof(ptree));
-  if (_mi_ft_parse(&ptree, info, keynr, record,0))
+  if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))
     DBUG_RETURN(NULL);
 
   DBUG_RETURN(ft_linearize(&ptree));
diff --git a/storage/myisam/ftdefs.h b/storage/myisam/ftdefs.h
index 11a283e0eb3..7c84c312a0b 100644
--- a/storage/myisam/ftdefs.h
+++ b/storage/myisam/ftdefs.h
@@ -119,10 +119,12 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
 uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
 
 void ft_parse_init(TREE *, CHARSET_INFO *);
-int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser);
+int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser,
+             MYSQL_FTPARSER_PARAM *param);
 FT_WORD * ft_linearize(TREE *);
 FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
-uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool);
+uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool,
+                  MYSQL_FTPARSER_PARAM *param);
 
 FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *);
 FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *);
@@ -142,4 +144,6 @@ void ft_boolean_close_search(FT_INFO *);
 float ft_boolean_get_relevance(FT_INFO *);
 my_off_t ft_boolean_get_docid(FT_INFO *);
 void ft_boolean_reinit_search(FT_INFO *);
-
+extern MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
+                                                       uint keynr);
+extern void ftparser_call_deinitializer(MI_INFO *info);
diff --git a/storage/myisam/mi_close.c b/storage/myisam/mi_close.c
index 39a91371323..9865ac72d62 100644
--- a/storage/myisam/mi_close.c
+++ b/storage/myisam/mi_close.c
@@ -105,6 +105,11 @@ int mi_close(register MI_INFO *info)
     my_free((gptr) info->s,MYF(0));
   }
   pthread_mutex_unlock(&THR_LOCK_myisam);
+  if (info->ftparser_param)
+  {
+    my_free((gptr)info->ftparser_param, MYF(0));
+    info->ftparser_param= 0;
+  }
   if (info->dfile >= 0 && my_close(info->dfile,MYF(0)))
     error = my_errno;
 
diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c
index 1a17febe94a..d158a3cb8e5 100644
--- a/storage/myisam/mi_create.c
+++ b/storage/myisam/mi_create.c
@@ -183,6 +183,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   }
   if (flags & HA_CREATE_DELAY_KEY_WRITE)
     options|= HA_OPTION_DELAY_KEY_WRITE;
+  if (flags & HA_CREATE_RELIES_ON_SQL_LAYER)
+    options|= HA_OPTION_RELIES_ON_SQL_LAYER;
 
   packed=(packed+7)/8;
   if (pack_reclength != INT_MAX32)
diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c
index 5d871aa1e38..c7f8c8d6e7f 100644
--- a/storage/myisam/mi_locking.c
+++ b/storage/myisam/mi_locking.c
@@ -21,7 +21,7 @@
   isamdatabase.
 */
 
-#include "myisamdef.h"
+#include "ftdefs.h"
 
 	/* lock table by F_UNLCK, F_RDLCK or F_WRLCK */
 
@@ -55,6 +55,7 @@ int mi_lock_database(MI_INFO *info, int lock_type)
   {
     switch (lock_type) {
     case F_UNLCK:
+      ftparser_call_deinitializer(info);
       if (info->lock_type == F_RDLCK)
 	count= --share->r_locks;
       else
diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c
index a09e4514d63..3f7c59293ba 100644
--- a/storage/myisam/mi_open.c
+++ b/storage/myisam/mi_open.c
@@ -141,12 +141,20 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	~(HA_OPTION_PACK_RECORD | HA_OPTION_PACK_KEYS |
 	  HA_OPTION_COMPRESS_RECORD | HA_OPTION_READ_ONLY_DATA |
 	  HA_OPTION_TEMP_COMPRESS_RECORD | HA_OPTION_CHECKSUM |
-	  HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE))
+          HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE |
+          HA_OPTION_RELIES_ON_SQL_LAYER))
     {
       DBUG_PRINT("error",("wrong options: 0x%lx", share->options));
       my_errno=HA_ERR_OLD_FILE;
       goto err;
     }
+    if ((share->options & HA_OPTION_RELIES_ON_SQL_LAYER) &&
+        ! (open_flags & HA_OPEN_FROM_SQL_LAYER))
+    {
+      DBUG_PRINT("error", ("table cannot be openned from non-sql layer"));
+      my_errno= HA_ERR_UNSUPPORTED;
+      goto err;
+    }
     /* Don't call realpath() if the name can't be a link */
     if (!strcmp(name_buff, org_name) ||
         my_readlink(index_name, org_name, MYF(0)) == -1)
@@ -418,6 +426,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	pos->flag=0;
 	pos++;
       }
+      share->ftparsers= 0;
     }
 
     disk_pos_assert(disk_pos + share->base.fields *MI_COLUMNDEF_SIZE, end_pos);
@@ -1051,6 +1060,7 @@ char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef)
    keydef->underflow_block_length=keydef->block_length/3;
    keydef->version	= 0;			/* Not saved */
    keydef->parser       = &ft_default_parser;
+   keydef->ftparser_nr  = 0;
    return ptr;
 }
 
diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h
index 564acb82e69..4e5c6be68fd 100644
--- a/storage/myisam/myisamdef.h
+++ b/storage/myisam/myisamdef.h
@@ -191,6 +191,7 @@ typedef struct st_mi_isam_share {	/* Shared between opens */
   ulong state_diff_length;
   uint	rec_reflength;			/* rec_reflength in use now */
   uint  unique_name_length;
+  uint32 ftparsers;                     /* Number of distinct ftparsers + 1 */
   File	kfile;				/* Shared keyfile */
   File	data_file;			/* Shared data file */
   int	mode;				/* mode of file on open */
@@ -231,6 +232,7 @@ struct st_myisam_info {
   /* accumulate indexfile changes between write's */
   TREE	        *bulk_insert;
   DYNAMIC_ARRAY *ft1_to_ft2;            /* used only in ft1->ft2 conversion */
+  MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit  */
   char *filename;			/* parameter to open filename */
   uchar *buff,				/* Temp area for key */
 	*lastkey,*lastkey2;		/* Last used search key */