query expansion for fulltext search

myisam/ft_boolean_search.c:
  assert.h and queues.h moved to ftdefs.h
myisam/ft_parser.c:
  ft_parse() with alloc
myisam/ft_static.c:
  api changes, ft_max_word_len_for_sort variable removed
myisam/ft_update.c:
  ft_parse() with alloc
myisam/ftdefs.h:
  variable ft_max_word_len_for_sort -> define FT_MAX_WORD_LEN_FOR_SORT
  api changes, ft_max_word_len_for_sort variable removed
  ft_parse() with alloc
myisam/mi_check.c:
  variable ft_max_word_len_for_sort -> define FT_MAX_WORD_LEN_FOR_SORT
myisam/myisamchk.c:
  ft_max_word_len_for_sort removed
mysql-test/r/fulltext.result:
  query expansion tests
mysql-test/r/fulltext_var.result:
  ft_max_word_len_for_sort removed
mysql-test/t/fulltext.test:
  query expansion tests
sql/ha_myisam.h:
  ft api changes for query expansion
sql/mysqld.cc:
  ft_max_word_len_for_sort removed
  ft_query_expansion_limit added
sql/set_var.cc:
  ft_max_word_len_for_sort removed
  ft_query_expansion_limit added
sql/sql_yacc.yy:
  EXPANSION added to keyword: rule

1 files changed, 47 insertions, 15 deletions

diff --git a/myisam/ft_nlq_search.c b/myisam/ft_nlq_search.c
index efe735c3d16..a38c52704f7 100644
--- a/myisam/ft_nlq_search.c
+++ b/myisam/ft_nlq_search.c
@@ -167,17 +167,28 @@ static int walk_and_copy(FT_SUPERDOC *from,
   DBUG_RETURN(0);
 }
 
+static int walk_and_push(FT_SUPERDOC *from,
+			 uint32 count __attribute__((unused)), QUEUE *best)
+{
+  DBUG_ENTER("walk_and_copy");
+  from->doc.weight+=from->tmp_weight*from->word_ptr->weight;
+  set_if_smaller(best->elements, ft_query_expansion_limit-1)
+  queue_insert(best, (byte *)& from->doc);
+  DBUG_RETURN(0);
+}
 
-static int FT_DOC_cmp(FT_DOC *a, FT_DOC *b)
+
+static int FT_DOC_cmp(void *unused __attribute__((unused)),
+                      FT_DOC *a, FT_DOC *b)
 {
   return sgn(b->weight - a->weight);
 }
 
 
 FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
-			    uint query_len, uint flags)
+			    uint query_len, uint flags, byte *record)
 {
-  TREE	     allocated_wtree, *wtree=&allocated_wtree;
+  TREE	      wtree;
   ALL_IN_ONE  aio;
   FT_DOC     *dptr;
   FT_INFO    *dlist=NULL;
@@ -196,24 +207,47 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
   aio.charset=info->s->keyinfo[keynr].seg->charset;
   aio.keybuff=info->lastkey+info->s->base.max_key_length;
 
-  bzero(&allocated_wtree,sizeof(allocated_wtree));
+  bzero(&wtree,sizeof(wtree));
 
   init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
             NULL, NULL);
 
-  ft_parse_init(&allocated_wtree, aio.charset);
-  if (ft_parse(&allocated_wtree,query,query_len))
+  ft_parse_init(&wtree, aio.charset);
+  if (ft_parse(&wtree,query,query_len,0))
     goto err;
 
-  if (tree_walk(wtree, (tree_walk_action)&walk_and_match, &aio,
+  if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
 		left_root_right))
-    goto err2;
+    goto err;
+
+  if (flags & FT_EXPAND && ft_query_expansion_limit)
+  {
+    QUEUE best;
+    init_queue(&best,ft_query_expansion_limit,0,0, &FT_DOC_cmp, 0);
+    tree_walk(&aio.dtree, (tree_walk_action) &walk_and_push,
+              &best, left_root_right);
+    while (best.elements)
+    {
+      my_off_t docid=((FT_DOC *)queue_remove(& best, 0))->dpos;
+      if (!(*info->read_record)(info,docid,record))
+      {
+        info->update|= HA_STATE_AKTIV;
+        _mi_ft_parse(&wtree, info, keynr, record,1);
+      }
+    }
+    delete_queue(&best);
+    reset_tree(&aio.dtree);
+    if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
+                  left_root_right))
+      goto err;
+
+  }
 
   dlist=(FT_INFO *)my_malloc(sizeof(FT_INFO)+
 			     sizeof(FT_DOC)*(aio.dtree.elements_in_tree-1),
 			     MYF(0));
-  if(!dlist)
-    goto err2;
+  if (!dlist)
+    goto err;
 
   dlist->please= (struct _ft_vft *) & _ft_vft_nlq;
   dlist->ndocs=aio.dtree.elements_in_tree;
@@ -225,13 +259,11 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
 	    &dptr, left_root_right);
 
   if (flags & FT_SORTED)
-    qsort(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort_cmp)&FT_DOC_cmp);
-
-err2:
-  delete_tree(wtree);
-  delete_tree(&aio.dtree);
+    qsort2(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort2_cmp)&FT_DOC_cmp, 0);
 
 err:
+  delete_tree(&aio.dtree);
+  delete_tree(&wtree);
   info->lastpos=saved_lastpos;
   DBUG_RETURN(dlist);
 }